added counts to articles, topics, ui fixes

vipra-cmd/src/main/java/de/vipra/cmd/lda/Analyzer.java

@@ -344,18 +344,6 @@ public class Analyzer {
 			newTopic.setRisingDecayRelevance(risingDecayRelevance);
 		}
 
-		// recreate windows, sequences and topics
-
-		final QueryBuilder builder = QueryBuilder.builder().criteria("topicModel", new TopicModel(modelConfig.getName()));
-
-		dbWindows.deleteMultiple(builder);
-		dbSequences.deleteMultiple(builder);
-		dbTopics.deleteMultiple(builder);
-
-		dbWindows.createMultiple(newWindows);
-		dbSequences.createMultiple(newSequences);
-		dbTopics.createMultiple(newTopics);
-
 		// create topic references and store document similarities
 
 		int idxArticle = -1;
@@ -369,10 +357,11 @@ public class Analyzer {
 			double reducedShare = 0;
 			final List<TopicShare> newTopicRefs = new ArrayList<>(topicCount);
 			for (int idxTopic = 0; idxTopic < topicCount; idxTopic++) {
-				if (topicDistribution[idxTopic] > 0.01) {
+				if (topicDistribution[idxTopic] >= modelConfig.getMinTopicShare()) {
 					reducedShare += topicDistribution[idxTopic];
 					final TopicShare newTopicRef = new TopicShare();
 					final TopicFull topicFull = newTopics.get(idxTopic);
+					topicFull.setArticlesCount(topicFull.getArticlesCount() + 1);
 					newTopicRef.setTopic(new Topic(topicFull.getId()));
 					newTopicRef.setShare(topicDistribution[idxTopic]);
 					newTopicRefs.add(newTopicRef);
@@ -419,13 +408,25 @@ public class Analyzer {
 				article.setSimilarArticles(similarArticles);
 
 				try {
-					dbArticles.updateSingle(article, "topicModel", "topics", "similarArticles");
+					dbArticles.updateSingle(article, "topicModel", "topics", "topicsCount", "similarArticles");
 				} catch (final DatabaseException e) {
 					ConsoleUtils.error(e);
 				}
 			}
 		}
 
+		// recreate entities
+
+		final QueryBuilder builder = QueryBuilder.builder().criteria("topicModel", new TopicModel(modelConfig.getName()));
+
+		dbWindows.deleteMultiple(builder);
+		dbSequences.deleteMultiple(builder);
+		dbTopics.deleteMultiple(builder);
+
+		dbWindows.createMultiple(newWindows);
+		dbSequences.createMultiple(newSequences);
+		dbTopics.createMultiple(newTopics);
+
 		topicModel.setLastGenerated(new Date());
 		dbTopicModels.replaceSingle(topicModel);
 	}

vipra-ui/app/html/articles/index.html

@@ -16,6 +16,7 @@
             <li value="title" class="nya-bs-option"><a>Title</a></li>
             <li value="date" class="nya-bs-option"><a>Date</a></li>
             <li value="created" class="nya-bs-option"><a>Added</a></li>
+            <li value="topicsCount" class="nya-bs-option"><a># of topics</a></li>
           </ol>
           <sort-dir ng-model="articlesIndexModels.sortdir" />
         </span>
@@ -25,6 +26,7 @@
             <tr ng-repeat="article in articles">
               <td>
                 <a ui-sref="articles.show({id: article.id})" ng-bind="::article.title"></a>
+                <span class="badge pull-right" ng-bind="::article.topicsCount" ng-attr-title="{{::article.topicsCount}} topic(s)"></span>
               </td>
             </tr>
           </tbody>

vipra-ui/app/html/articles/show.html

@@ -42,21 +42,21 @@
                 </tr>
                 <tr>
                   <th>Word count</th>
-                  <td ng-bind="::article.stats.wordCount"></td>
+                  <td ng-bind-template="{{::article.stats.wordCount}} ({{::article.stats.processedWordCount}} &darr; {{::Vipra.toPercent(article.stats.reductionRatio)}}%)" ng-attr-title="{{::article.stats.wordCount}} words in this article, {{::article.stats.processedWordCount}} after cleaning ({{::Vipra.toPercent(article.stats.reductionRatio)}}% reduction)"></td>
                 </tr>
               </tbody>
             </table>
             <h3>Topics</h3>
-            <table class="table table-bordered table-condensed">
+            <table class="table table-bordered table-condensed" ng-show="article.topics.length">
               <thead>
                 <tr>
                   <th class="infocol" ng-model="articlesShowModels.topicsSort" sort-by="share">Share</th>
-                  <th ng-model="articlesShowModels.topicsSort" sort-by="name">Name</th>
+                  <th ng-model="articlesShowModels.topicsSort" sort-by="topic.name">Name</th>
                   <th style="width:1px"></th>
                 </tr>
               </thead>
               <tbody>
-                <tr ng-repeat="topic in article.topics | orderBy:articlesShowModels.topicsSort">
+                <tr ng-repeat="topic in article.topics | orderBy:articlesShowModels.topicsSort" ng-mouseenter="highlightSlice(topic.topic.id, true)" ng-mouseleave="highlightSlice(topic.topic.id, false)">
                   <td class="text-right" ng-bind-template="{{(topic.share*100).toFixed(0)}}%"></td>
                   <td>
                     <topic-link topic="topic.topic" />
@@ -67,7 +67,7 @@
                 </tr>
               </tbody>
             </table>
-            <span class="text-muted" ng-hide="article.topics.length > 0">No topics</span>
+            <p class="text-muted" ng-hide="article.topics.length">No topics</p>
           </div>
           <div class="col-md-4">
             <h3>Share</h3>
@@ -75,22 +75,23 @@
           </div>
         </div>
         <h3>Similar articles</h3>
-        <table class="table table-bordered table-condensed">
+        <table class="table table-bordered table-condensed" ng-show="article.similarArticles.length">
           <thead>
             <tr>
-              <th class="infocol" ng-model="articlesShowModels.similarSort" sort-by="divergence">Share</th>
+              <th class="infocol" ng-model="articlesShowModels.similarSort" sort-by="share">Share</th>
               <th ng-model="articlesShowModels.similarSort" sort-by="article.title">Title</th>
             </tr>
           </thead>
           <tbody>
             <tr ng-repeat="simArticle in article.similarArticles | orderBy:articlesShowModels.similarSort">
-              <td class="text-right" ng-bind-template="{{((1-simArticle.divergence)*100).toFixed(0)}}%"></td>
+              <td class="text-right" ng-bind-template="{{::simArticle.share}}%"></td>
               <td>
                 <a ui-sref="articles.show({id: simArticle.article.id})" ng-attr-title="{{::simArticle.article.title}}" ng-bind="::simArticle.article.title"></a>
               </td>
             </tr>
           </tbody>
         </table>
+        <p class="text-muted" ng-hide="article.similarArticles.length">No similar articles.</p>
         <hr>
         <div class="text-justify" ng-bind-html="::article.text"></div>
       </div>
@@ -100,25 +101,33 @@
           <div class="col-md-12">
             <div class="panel panel-default">
               <div class="panel-heading">
-                Found
-                <ng-pluralize count="words.length||0" when="{0:'no words',1:'1 word',other:'{} words'}"></ng-pluralize> in the database.
+                Found <ng-pluralize count="allWords.length||0" when="{0:'no words',1:'1 word',other:'{} unique words'}"></ng-pluralize> for this article.<br>
+                Article has <ng-pluralize count="article.stats.wordCount||0" when="{0:'no words',1:'1 word',other:'{} words'}"></ng-pluralize>, <span ng-bind-template="{{::article.stats.processedWordCount}} after cleaning ({{::Vipra.toPercent(article.stats.reductionRatio)}}% reduction)"></span>.
               </div>
-              <table class="table table-bordered table-condensed">
+              <table class="table table-bordered table-condensed table-fixed">
                 <thead>
                   <tr>
                     <th ng-model="articlesShowModels.wordsSort" sort-by="word">Word</th>
                     <th ng-model="articlesShowModels.wordsSort" sort-by="count">Count</th>
+                    <th>Share</th>
+                    <th>Reduced share</th>
                   </tr>
                 </thead>
                 <tbody>
                   <tr ng-repeat="word in words | orderBy:articlesShowModels.wordsSort">
                     <td>
-                      <word-link word="word" />
+                      <word-link word="::word" />
                     </td>
-                    <td ng-bind="word.count"></td>
+                    <td ng-bind="::word.count"></td>
+                    <td ng-bind-template="{{::Vipra.toPercent(word.count/article.stats.wordCount, 2)}}%"></td>
+                    <td ng-bind-template="{{::Vipra.toPercent(word.count/article.stats.processedWordCount, 2)}}%"></td>
                   </tr>
                 </tbody>
               </table>
+              <div class="panel-footer">
+                <ng-pluralize count="words.length" when="{0:'No words',1:'First word',other:'First {} words'}"></ng-pluralize>.
+                <button class="btn btn-default btn-sm" ng-click="showMoreWords()" ng-show="words.length<allWords.length">Show more</button>
+              </div>
             </div>
           </div>
         </div>

vipra-ui/app/html/partials/topic-popover.html

-<table class="table table-bordered table-condensed table-nomargin">
+<table class="table table-bordered table-condensed nomargin">
   <tbody>
     <tr>
       <th class="text-center">&#956;</th>

vipra-ui/app/html/topics/index.html

@@ -24,6 +24,7 @@
             <tr ng-repeat="topic in topics">
               <td>
                 <topic-link topic="topic" />
+                <span class="badge pull-right" ng-bind="::topic.articlesCount" ng-attr-title="{{::topic.articlesCount}} article(s)"></span>
               </td>
             </tr>
           </tbody>

vipra-ui/app/html/topics/show.html

@@ -113,7 +113,7 @@
             <small>Sequence:</small>
             <sequence-dropdown ng-model="sequenceId" sequences="topic.sequences"></sequence-dropdown>
           </div>
-          <table class="table table-condensed table-bordered table-hover" ng-show="sequence">
+          <table class="table table-condensed table-bordered table-hover table-fixed" ng-show="sequence">
             <thead>
               <tr>
                 <th ng-model="topicsShowModels.seqSortWords" sort-by="word">Word</th>

vipra-ui/app/index.html

@@ -98,7 +98,7 @@
           <h4 class="modal-title">Topic Models</h4>
         </div>
         <div class="modal-body">
-          <ul class="list-group" ng-show="topicModels.length">
+          <ul class="list-group nomargin" ng-show="topicModels.length">
             <button type="button" class="list-group-item topic-model" ng-repeat="topicModel in topicModels" ng-click="changeTopicModel(topicModel)" ng-class="{'active selected-model':rootModels.topicModel.id===topicModel.id}">
               <span class="badge" ng-bind="topicModel.articleCount" ng-show="topicModel.articleCount" ng-attr-title="{{topicModel.articleCount + ' article(s)'}}"></span>
               <span class="badge" ng-bind="topicModel.topicCount" ng-show="topicModel.topicCount" ng-attr-title="{{topicModel.topicCount + ' topic(s)'}}"></span>
@@ -114,6 +114,8 @@
           <p ng-hide="topicModels.length || loading.any">
             No topic models in the database. Create a topic model and import data into it to begin.
           </p>
+        </div>
+        <div class="modal-body">
           <h4>Quick start</h4>
           <ol>
             <li>

vipra-ui/app/js/controllers.js

@@ -71,7 +71,7 @@
         combo: 'e',
         description: 'Go to explorer',
         callback: function() {
-          if ($state.current.name !== 'explorer')
+          if ($scope.rootModels.topicModel && $state.current.name !== 'explorer')
             $state.transitionTo('explorer');
         }
       });
@@ -80,7 +80,7 @@
         combo: 'a',
         description: 'Go to articles',
         callback: function() {
-          if ($state.current.name !== 'articles')
+          if ($scope.rootModels.topicModel && $state.current.name !== 'articles')
             $state.transitionTo('articles');
         }
       });
@@ -89,7 +89,7 @@
         combo: 't',
         description: 'Go to topics',
         callback: function() {
-          if ($state.current.name !== 'topics')
+          if ($scope.rootModels.topicModel && $state.current.name !== 'topics')
             $state.transitionTo('topics');
         }
       });
@@ -597,8 +597,8 @@
     function($scope, $state, $stateParams, $timeout, ArticleFactory) {
 
       $scope.articlesShowModels = {
-        topicsSort: 'share',
-        similarSort: 'divergence',
+        topicsSort: '-share',
+        similarSort: '-share',
         wordsSort: '-count'
       };
 
@@ -611,6 +611,12 @@
         $scope.articleCreated = Vipra.formatDateTime($scope.article.created);
         $scope.articleModified = Vipra.formatDateTime($scope.article.modified);
 
+        // calculate share from divergence
+        if($scope.article.similarArticles) {
+          for(var i = 0; i < $scope.article.similarArticles.length; i++)
+            $scope.article.similarArticles[i].share = ((1 - $scope.article.similarArticles[i].divergence) * 100).toFixed(0);
+        }
+
         // take topic model from article
         if (!angular.isObject($scope.rootModels.topicModel))
           $scope.rootModels.topicModel = data.topicModel;
@@ -626,7 +632,8 @@
             d = {
               name: topics[i].topic.name,
               y: topics[i].share,
-              color: colors[i]
+              color: colors[i],
+              id: topics[i].topic.id
             };
 
             topicShareSeries.push(d);
@@ -656,9 +663,31 @@
           id: $stateParams.id,
           fields: 'words'
         }, function(data) {
-          $scope.words = data.words;
+          $scope.allWords = data.words;
+          $scope.showMoreWords();
         });
       };
+
+      var wordsCount = 0;
+      $scope.showMoreWords = function() {
+        wordsCount += 20;
+        $scope.words = $scope.allWords.slice(0, wordsCount);
+      };
+
+      var topicShareChartElement = $('#topic-share');
+      $scope.highlightSlice = function(id, toggle) {
+        var highcharts = topicShareChartElement.highcharts();
+        if (!highcharts) return;
+        var point = highcharts.get(id);
+        if (!point) return;
+
+        if (toggle) {
+          point.onMouseOver();
+        } else {
+          point.onMouseOut();
+          highcharts.tooltip.hide();
+        }
+      };
     }
   ]);
 

vipra-ui/app/js/helpers.js

@@ -22,10 +22,12 @@
     return date.toLocaleDateString() + " " + date.toLocaleTimeString();
   };
 
-  Vipra.toPercent = function(input) {
+  Vipra.toPercent = function(input, nums) {
+    if (typeof input === 'undefined')
+      return;
     if (typeof input !== 'number')
       input = parseInt(input, 10);
-    return Math.round(input * 100);
+    return (input * 100).toFixed(isNaN(nums) ? 0 : nums);
   };
 
   Vipra.createInitial = function(text) {

vipra-ui/app/less/app.less

@@ -19,6 +19,10 @@ a:hover {
   cursor: pointer;
 }
 
+[sort-by]:hover {
+  background: #f5f5f5;
+}
+
 .heading {
   height: 125px;
   margin: 25px 0;
@@ -144,7 +148,7 @@ a:hover {
   table-layout: fixed;
 }
 
-.table-nomargin {
+.nomargin {
   margin: 0;
 }
 
@@ -420,6 +424,10 @@ word-menu {
   }
 }
 
+.modal-body + .modal-body {
+  border-top: 1px solid #e5e5e5;
+}
+
 @-moz-keyframes spin {
   100% {
     -moz-transform: rotateY(360deg);

vipra-util/src/main/java/de/vipra/util/Constants.java

@@ -84,6 +84,12 @@ public class Constants {
 	 */
 	public static final double RISING_DECAY_LAMBDA = 0.0;
 
+	/**
+	 * Minimum topic share for an article. Topics with a smaller share are
+	 * ignored.
+	 */
+	public static final double MIN_TOPIC_SHARE = 0.01;
+
 	/**
 	 * Minimum probability of words. Words with lower probability are ignored.
 	 * Default 0.01.

vipra-util/src/main/java/de/vipra/util/model/ArticleFull.java

@@ -62,6 +62,8 @@ public class ArticleFull implements Model<ObjectId>, Serializable {
 	@QueryIgnore(multi = true)
 	private List<TopicShare> topics;
 
+	private int topicsCount;
+
 	@Embedded
 	@QueryIgnore(multi = true)
 	private List<SimilarArticle> similarArticles;
@@ -173,6 +175,11 @@ public class ArticleFull implements Model<ObjectId>, Serializable {
 
 	public void setTopics(final List<TopicShare> topics) {
 		this.topics = topics;
+		this.topicsCount = topics == null ? 0 : topics.size();
+	}
+
+	public int getTopicsCount() {
+		return topicsCount;
 	}
 
 	@ElasticIndex("topics")

vipra-util/src/main/java/de/vipra/util/model/TopicFull.java

@@ -58,6 +58,8 @@ public class TopicFull implements Model<ObjectId>, Serializable {
 	@QueryIgnore(multi = true)
 	private Double risingDecayRelevance;
 
+	private int articlesCount;
+
 	private Date created;
 
 	private Date modified;
@@ -148,6 +150,14 @@ public class TopicFull implements Model<ObjectId>, Serializable {
 		this.risingDecayRelevance = risingDecayRelevance;
 	}
 
+	public int getArticlesCount() {
+		return articlesCount;
+	}
+
+	public void setArticlesCount(int articlesCount) {
+		this.articlesCount = articlesCount;
+	}
+
 	public Date getCreated() {
 		return created;
 	}

vipra-util/src/main/java/de/vipra/util/model/TopicModelConfig.java

@@ -29,6 +29,7 @@ public class TopicModelConfig implements Serializable {
 	private int documentMinimumWordFrequency = Constants.DOCUMENT_MIN_WORD_FREQ;
 	private int spotlightSupport = Constants.SPOTLIGHT_SUPPORT;
 	private double spotlightConfidence = Constants.SPOTLIGHT_CONFIDENCE;
+	private double minTopicShare = Constants.MIN_TOPIC_SHARE;
 	private double minRelativeProbability = Constants.MIN_RELATIVE_PROB;
 	private double risingDecayLambda = Constants.RISING_DECAY_LAMBDA;
 	private double maxSimilarDocumentsDivergence = Constants.MAX_SIMILAR_DOCUMENTS_DIVERGENCE;
@@ -49,6 +50,7 @@ public class TopicModelConfig implements Serializable {
 		documentMinimumWordFrequency = topicModelConfig.getDocumentMinimumWordFrequency();
 		spotlightSupport = topicModelConfig.getSpotlightSupport();
 		spotlightConfidence = topicModelConfig.getSpotlightConfidence();
+		minTopicShare = topicModelConfig.getMinTopicShare();
 		minRelativeProbability = topicModelConfig.getMinRelativeProbability();
 		risingDecayLambda = topicModelConfig.getRisingDecayLambda();
 		maxSimilarDocumentsDivergence = topicModelConfig.getMaxSimilarDocumentsDivergence();
@@ -160,6 +162,14 @@ public class TopicModelConfig implements Serializable {
 		this.spotlightConfidence = spotlightConfidence;
 	}
 
+	public double getMinTopicShare() {
+		return minTopicShare;
+	}
+
+	public void setMinTopicShare(double minTopicShare) {
+		this.minTopicShare = minTopicShare;
+	}
+
 	public double getMinRelativeProbability() {
 		return minRelativeProbability;
 	}