added data sql script and json transformation

moved vagrantfile up to access other dirs
updated bootstrap script to init mongo database, do various changes and install node+npm
added installation of mongo-express for mongodb web ui

vm/Vagrantfile → Vagrantfile

@@ -11,7 +11,8 @@ Vagrant.configure(2) do |config|
   	master.vm.hostname = 'cma-master'
     master.vm.network :private_network, ip: '192.168.10.10'
     master.vm.network :forwarded_port, guest: 27017, host: 27017
-  	master.vm.provision :shell, path: 'bootstrap.sh'
+    master.vm.network :forwarded_port, guest: 8081, host: 8081
+  	master.vm.provision :shell, path: 'vm/bootstrap.sh'
   end
 
 end

ma-impl.sublime-workspace

@@ -450,13 +450,14 @@
 	"expanded_folders":
 	[
 		"/home/eike/Repositories/fu/ss15/ma/impl",
-		"/home/eike/Repositories/fu/ss15/ma/impl/vm",
-		"/home/eike/Repositories/fu/ss15/ma/impl/vm/config"
+		"/home/eike/Repositories/fu/ss15/ma/impl/data"
 	],
 	"file_history":
 	[
-		"/home/eike/Repositories/fu/ss15/ma/impl/vm/Vagrantfile",
 		"/home/eike/Repositories/fu/ss15/ma/impl/vm/bootstrap.sh",
+		"/home/eike/Repositories/fu/ss15/ma/impl/data/data.json",
+		"/home/eike/Repositories/fu/ss15/ma/impl/vm/Vagrantfile",
+		"/home/eike/Repositories/fu/ss15/ma/impl/data/data.sql",
 		"/home/eike/Repositories/fu/ss15/ma/impl/vm/env.sh",
 		"/home/eike/Repositories/fu/ws1415/ntdb/Vagrantfile",
 		"/home/eike/Repositories/fu/ws1415/ntdb/puppet/manifests/default.pp",
@@ -555,6 +556,39 @@
 		"case_sensitive": false,
 		"find_history":
 		[
+			"000+00\"",
+			"\\$date\": ([^,]+),",
+			"\"id\"",
+			"\\$date\": \"([^\"]+)\"",
+			"\"date\"",
+			"",
+			"new Date\\(([^\\)]+)\\)",
+			"\"text\": \"  ",
+			"\\\"},",
+			"\\\", \"url\": \\\"",
+			"\\\"), \"text\": \\\"",
+			"new Date(\\\"",
+			"\"title\": \\\"",
+			"\\\", \"title",
+			"\\\"url\\\":",
+			"\\\"title\\\":",
+			"\\\"text\\\":",
+			"\\\"date\\\":",
+			"\\\"id\\\":",
+			"\"",
+			"\"text\": \" ",
+			"title: ",
+			"url: ",
+			"text: ",
+			"date: ",
+			"\\((\\d+),\\s+'(.*?)',\\s+'(.*?)',\\s+'(.*?)',\\s+'(.*?)'\\).",
+			"INSERT INTO.*\\n",
+			");",
+			"\\);$",
+			"^\\(",
+			"INSERT INTO.*\\n",
+			"INSERT INTO",
+			"INSERT INTO.*\\n",
 			"thumbnail",
 			"Gesellscha",
 			"http://.*?\"",
@@ -649,47 +683,45 @@
 			"account.name",
 			"category.name",
 			"t('",
-			".name",
-			"pluralize",
-			"balance",
-			"balances",
-			"balance",
-			"bal",
-			"balance",
-			"datetime-js",
-			"datetime-rb",
-			"datetime-js",
-			"archit",
-			"banner-padding",
-			"active",
-			"should_get_create",
-			"password_confirmation",
-			"assets.compile",
-			"assets.enabled",
-			"load_config",
-			"class:",
-			"help",
-			"compress",
-			"reset_layout",
-			"apply_lay",
-			"Startpages",
-			"StartpagesController",
-			"pull-center",
-			"end",
-			"../fonts/",
-			"halfl",
-			"login_path",
-			"overf",
-			"email",
-			"$user->email",
-			"<br/>"
+			".name"
 		],
 		"highlight": true,
 		"in_selection": false,
 		"preserve_case": false,
-		"regex": true,
+		"regex": false,
 		"replace_history":
 		[
+			"000+0000\"",
+			"date\": { \"\\$date\": $1 },",
+			"\"_id\"",
+			"\\$date\": \"$1T00:00:00.000+00\"",
+			"\"$date\"",
+			"",
+			"$1",
+			"\"text\": \"",
+			"\"},",
+			"\", \"url\": \"",
+			"\"), \"text\": \"",
+			"new Date(\"",
+			"\"title\": \"",
+			"\", \"title",
+			"\"url\":",
+			"\"title\":",
+			"\"text\":",
+			"\"date\":",
+			"\"id\":",
+			"\\\"",
+			"\"text\": \"",
+			"\"title\": ",
+			"\"url\": ",
+			"\"text\": ",
+			"\"date\": ",
+			"{\"id\": $1, date: new Date(\"$2\"), text: \"$3\", title: \"$4\", url: \"$5\"},",
+			"",
+			"},",
+			"}.",
+			"{id: ",
+			"",
 			"thumb",
 			"",
 			"\\textbf{",
@@ -786,25 +818,7 @@
 			"Lang::t",
 			"@__",
 			"@T",
-			"'",
-			"\\t",
-			"'$1'",
-			"$1",
-			"",
-			"url('../assets",
-			"'",
-			"    ",
-			"",
-			"    ",
-			"\\t",
-			"$transaction",
-			"<?",
-			"<?=",
-			"\\t",
-			"public/",
-			"resources/",
-			"",
-			"\\t"
+			"'"
 		],
 		"reverse": false,
 		"show_context": true,

vm/bootstrap.sh

 #!/bin/sh
 
+# -----------------------------------------------------------------------------
 # upgrade system
 apt-get update
 apt-get upgrade -y
-apt-get install git scala -y
+apt-get install git scala npm libkrb5-dev -y
 
+# -----------------------------------------------------------------------------
+# fix node setup in ubuntu
+ln -s $(which nodejs) /bin/node
+
+# -----------------------------------------------------------------------------
 # install & start mongodb
 apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv 7F0CEB10
 echo "deb http://repo.mongodb.org/apt/ubuntu trusty/mongodb-org/3.0 multiverse" | tee /etc/apt/sources.list.d/mongodb-org-3.0.list
 apt-get update
 apt-get install -y mongodb-org
-cp /vagrant/config/master-mongod.conf /etc/mongod.conf
+
+# copy configuration
+cp /vagrant/vm/config/master-mongod.conf /etc/mongod.conf
+
+# disable hugepages (https://docs.mongodb.org/manual/tutorial/transparent-huge-pages/)
+cp /vagrant/vm/config/disable-transparent-hugepages /etc/init.d/
+chmod 755 /etc/init.d/disable-transparent-hugepages
+update-rc.d disable-transparent-hugepages defaults
+echo 'never' > /sys/kernel/mm/transparent_hugepage/enabled 
+echo 'never' > /sys/kernel/mm/transparent_hugepage/defrag
+
+# install node, mongo-express
+apt-get install npm libkrb5-dev
+npm install -g mongo-express forever
+cp /vagrant/vm/config/mongo-express.config.js /usr/local/lib/node_modules/mongo-express/config.js
+
+# restart service
 service mongod restart
 
+# import mongodb data
+mongoimport --db test --collection articles --file /vagrant/data/data.json --jsonArray
+
+# start web ui
+forever start $(which mongo-express)
+
+# -----------------------------------------------------------------------------
 # install spark
 wget ftp://mirror.netcologne.de/apache.org/spark/spark-1.5.2/spark-1.5.2-bin-hadoop2.6.tgz
 tar zxf spark-1.5.2-bin-hadoop2.6.tgz
 rm spark-1.5.2-bin-hadoop2.6.tgz
 mv spark-1.5.2-bin-hadoop2.6 spark
-echo "source /vagrant/env.sh" >> /home/vagrant/.bashrc
+echo "source /vagrant/vm/env.sh" >> /home/vagrant/.bashrc
 
+# -----------------------------------------------------------------------------
 # disable firewall
 ufw disable
\ No newline at end of file

vm/config/disable-transparent-hugepages

+#!/bin/sh
+### BEGIN INIT INFO
+# Provides:          disable-transparent-hugepages
+# Required-Start:    $local_fs
+# Required-Stop:
+# X-Start-Before:    mongod mongodb-mms-automation-agent
+# Default-Start:     2 3 4 5
+# Default-Stop:      0 1 6
+# Short-Description: Disable Linux transparent huge pages
+# Description:       Disable Linux transparent huge pages, to improve
+#                    database performance.
+### END INIT INFO
+
+case $1 in
+  start)
+    if [ -d /sys/kernel/mm/transparent_hugepage ]; then
+      thp_path=/sys/kernel/mm/transparent_hugepage
+    elif [ -d /sys/kernel/mm/redhat_transparent_hugepage ]; then
+      thp_path=/sys/kernel/mm/redhat_transparent_hugepage
+    else
+      return 0
+    fi
+
+    echo 'never' > ${thp_path}/enabled
+    echo 'never' > ${thp_path}/defrag
+
+    unset thp_path
+    ;;
+esac
\ No newline at end of file

vm/config/mongo-express.config.js

+'use strict';
+
+var url = require('url');
+
+if (typeof process.env.MONGODB_PORT === 'string') {
+  var mongoConnection = url.parse(process.env.MONGODB_PORT);
+  process.env.ME_CONFIG_MONGODB_SERVER = mongoConnection.hostname;
+  process.env.ME_CONFIG_MONGODB_PORT = mongoConnection.port;
+}
+
+module.exports = {
+  mongodb: {
+    server: process.env.ME_CONFIG_MONGODB_SERVER || 'localhost',
+    port: process.env.ME_CONFIG_MONGODB_PORT || 27017,
+
+    //autoReconnect: automatically reconnect if connection is lost
+    autoReconnect: true,
+    //poolSize: size of connection pool (number of connections to use)
+    poolSize: 4,
+    //set admin to true if you want to turn on admin features
+    //if admin is true, the auth list below will be ignored
+    //if admin is true, you will need to enter an admin username/password below (if it is needed)
+    admin: true,
+
+
+    // >>>>  If you are using regular accounts, fill out auth details in the section below
+    // >>>>  If you have admin auth, leave this section empty and skip to the next section
+    auth: [
+      /*
+       * Add the the name, the username, and the password of the databases you want to connect to
+       * Add as many databases as you want!
+      {
+        database: 'test',
+        username: 'user',
+        password: 'pass'
+      }
+      */
+      ],
+
+
+    //  >>>>  If you are using an admin mongodb account, or no admin account exists, fill out section below
+    //  >>>>  Using an admin account allows you to view and edit all databases, and view stats
+
+    //leave username and password empty if no admin account exists
+    adminUsername: process.env.ME_CONFIG_MONGODB_ADMINUSERNAME || '',
+    adminPassword: process.env.ME_CONFIG_MONGODB_ADMINPASSWORD || '',
+    //whitelist: hide all databases except the ones in this list  (empty list for no whitelist)
+    whitelist: [],
+    //blacklist: hide databases listed in the blacklist (empty list for no blacklist)
+    blacklist: []
+  },
+
+  site: {
+    host: '0.0.0.0',
+    port: 8081,
+    cookieSecret: process.env.ME_CONFIG_SITE_COOKIESECRET || 'cookiesecret',
+    sessionSecret: process.env.ME_CONFIG_SITE_SESSIONSECRET || 'sessionsecret',
+    cookieKeyName: 'mongo-express',
+    sslEnabled: process.env.ME_CONFIG_SITE_SSL_ENABLED || false,
+    sslCert: process.env.ME_CONFIG_SITE_SSL_CRT_PATH || '',
+    sslKey: process.env.ME_CONFIG_SITE_SSL_KEY_PATH || ''
+  },
+
+  //set useBasicAuth to true if you want to authehticate mongo-express loggins
+  //if admin is false, the basicAuthInfo list below will be ignored
+  //this will be true unless ME_CONFIG_BASICAUTH_USERNAME is set and is the empty string
+  useBasicAuth: process.env.ME_CONFIG_BASICAUTH_USERNAME !== '',
+
+  basicAuth: {
+    username: process.env.ME_CONFIG_BASICAUTH_USERNAME || 'admin',
+    password: process.env.ME_CONFIG_BASICAUTH_PASSWORD || 'pass'
+  },
+
+  options: {
+    //documentsPerPage: how many documents you want to see at once in collection view
+    documentsPerPage: 10,
+    //editorTheme: Name of the theme you want to use for displaying documents
+    //See http://codemirror.net/demo/theme.html for all examples
+    editorTheme: process.env.ME_CONFIG_OPTIONS_EDITORTHEME || 'rubyblue',
+
+    //The options below aren't being used yet
+
+    //cmdType: the type of command line you want mongo express to run
+    //values: eval, subprocess
+    //  eval - uses db.eval. commands block, so only use this if you have to
+    //  subprocess - spawns a mongo command line as a subprocess and pipes output to mongo express
+    cmdType: 'eval',
+    //subprocessTimeout: number of seconds of non-interaction before a subprocess is shut down
+    subprocessTimeout: 300,
+    //readOnly: if readOnly is true, components of writing are not visible.
+    readOnly: false
+  },
+
+  // Specify the default keyname that should be picked from a document to display in collections list.
+  // Keynames can be specified for every database and collection.
+  // If no keyname is specified, it defalts to '_id', which is a mandatory feild.
+  // For Example :
+  // defaultKeyNames{
+  //   "world_db":{  //Database Name
+  //     "continent":"cont_name", // collection:feild
+  //     "country":"country_name",
+  //     "city":"name"
+  //   }
+  // }
+  defaultKeyNames: {
+    "test": {
+      "articles": "title"
+    }
+  }
+};