updated vagrant box and provisioning

removed many errors and typos removed spark, scala, maven, mahout

updated vagrant box and provisioning
71a16b92 · Eike Cochu · c756609d · 71a16b92 · 71a16b92 · 71a16b92
Commit 71a16b92 authored 9 years ago by Eike Cochu
--- a/Vagrantfile
+++ b/Vagrantfile
@@ -12,12 +12,8 @@ Vagrant.configure(2) do |config|
    master.vm.network :private_network, ip: '192.168.10.10'
    master.vm.network :forwarded_port, guest: 27017, host: 27017 # MongoDB
    master.vm.network :forwarded_port, guest: 8081,  host: 8081  # MongoDB Express
-    master.vm.network :forwarded_port, guest: 8001,  host: 8001  # Spark MasterUI
-    master.vm.network :forwarded_port, guest: 8002,  host: 8002  # Spark WorkerUI
-    master.vm.network :forwarded_port, guest: 7077,  host: 7077  # Spark
    master.vm.network :forwarded_port, guest: 8080,  host: 8000  # Tomcat
    master.vm.network :forwarded_port, guest: 9200,  host: 9200  # ElasticSearch REST API
-    master.vm.provision :shell, path: 'vm/bootstrap.sh'
  end

 end
--- a/vm/bootstrap.sh
+++ b/vm/bootstrap.sh
@@ -6,8 +6,8 @@ CONFIG=$VMDIR/config

 # -----------------------------------------------------------------------------
 # add repos
-apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv 7F0CEB10
-echo "deb http://repo.mongodb.org/apt/ubuntu trusty/mongodb-org/3.0 multiverse" | tee /etc/apt/sources.list.d/mongodb-org-3.0.list
+apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv EA312927
+echo "deb http://repo.mongodb.org/apt/ubuntu trusty/mongodb-org/3.2 multiverse" | sudo tee /etc/apt/sources.list.d/mongodb-org-3.2.list

 # -----------------------------------------------------------------------------
 # upgrade system
@@ -18,27 +18,10 @@ apt-get upgrade -y
 # install packages
 apt-get install git gdebi-core openjdk-7-jdk openjdk-7-jre -y

-# -----------------------------------------------------------------------------
-# install scala
-wget http://www.scala-lang.org/files/archive/scala-2.10.6.deb
-gdebi -n scala-2.10.6.deb
-rm scala-2.10.6.deb
-
-# -----------------------------------------------------------------------------
-# install spark
-wget http://mirror.netcologne.de/apache.org/spark/spark-1.5.2/spark-1.5.2-bin-hadoop2.6.tgz
-tar zxf spark-1.5.2-bin-hadoop2.6.tgz
-rm spark-1.5.2-bin-hadoop2.6.tgz
-mv spark-1.5.2-bin-hadoop2.6 spark
-ln -sf $CONFIG/spark-env.sh /home/vagrant/spark/conf/spark-env.sh
-
 # -----------------------------------------------------------------------------
 # install mongodb
 apt-get install -y mongodb-org

-# copy configuration
-ln -sf $CONFIG/master-mongod.conf /etc/mongod.conf
-
 # disable hugepages (https://docs.mongodb.org/manual/tutorial/transparent-huge-pages/)
 cp $CONFIG/disable-transparent-hugepages /etc/init.d/
 chmod 755 /etc/init.d/disable-transparent-hugepages
@@ -49,9 +32,6 @@ echo 'never' > /sys/kernel/mm/transparent_hugepage/defrag
 # restart service
 service mongod restart

-# import mongodb data
-mongoimport --db test --collection articles --file $DATA/data.json --jsonArray
-
 # -----------------------------------------------------------------------------
 # install mongo express
 apt-get install npm libkrb5-dev
@@ -61,16 +41,19 @@ ln -sf $CONFIG/mongo-express.config.js /usr/local/lib/node_modules/mongo-express

 # -----------------------------------------------------------------------------
 # install tomcat
-wget http://mirror.netcologne.de/apache.org/tomcat/tomcat-8/v8.0.30/bin/apache-tomcat-8.0.30.tar.gz
-tar zxf apache-tomcat-8.0.30.tar.gz
-mv apache-tomcat-8.0.30 /usr/share/tomcat8
-ln -s /usr/share/tomcat /usr/share/tomcat8
-# setup server config
-rm /usr/share/tomcat/conf/server.xml /usr/share/tomcat/conf/tomcat-users.xml
-ln -s $CONFIG/tomcat-server.xml /usr/share/tomcat/conf/server.xml
-ln -s $CONFIG/tomcat-users.xml /usr/share/tomcat/conf/tomcat-users.xml
+
+TOMCAT_MAJOR="8"
+TOMCAT_MINOR="0.30"
+TOMCAT_VERSION="$TOMCAT_MAJOR.$TOMCAT_MINOR"
+TOMCAT_ROOT=/usr/share/tomcat$TOMCAT_MAJOR
+
+wget http://mirror.netcologne.de/apache.org/tomcat/tomcat-$TOMCAT_MAJOR/v$TOMCAT_VERSION/bin/apache-tomcat-$TOMCAT_VERSION.tar.gz
+tar zxf apache-tomcat-$TOMCAT_VERSION.tar.gz
+rm apache-tomcat-$TOMCAT_VERSION.tar.gz
+mv apache-tomcat-$TOMCAT_VERSION $TOMCAT_ROOT
+ln -s $TOMCAT_ROOT /usr/share/tomcat
 # set permissions
-chown -R vagrant:vagrant /usr/share/tomcat8
+chown -R vagrant:vagrant /usr/share/tomcat
 chmod +x /usr/share/tomcat/bin/*.sh
 # setup startup script
 cp $CONFIG/initd-tomcat /etc/init.d/tomcat
@@ -82,24 +65,12 @@ ln -s $VMDIR/webapps /usr/share/tomcat/webapps
 # start tomcat
 service tomcat start

-# -----------------------------------------------------------------------------
-# install maven
-wget http://mirror.netcologne.de/apache.org/maven/maven-3/3.3.9/binaries/apache-maven-3.3.9-bin.tar.gz
-tar zxf apache-maven-3.3.9-bin.tar.gz
-mv apache-maven-3.3.9-bin maven
-rm apache-maven-3.3.9-bin.tar.gz
-
-# -----------------------------------------------------------------------------
-# install mahout
-git clone https://github.com/apache/mahout.git mahout
-mvn -DskipTests -X clean -f ./mahout install
-
 # -----------------------------------------------------------------------------
 # install elasticsearch
 wget https://download.elasticsearch.org/elasticsearch/release/org/elasticsearch/distribution/deb/elasticsearch/2.1.1/elasticsearch-2.1.1.deb
-gdebi -n elasticsearch-2.1.0.deb
-rm elasticsearch-2.1.0.deb
-service start elasticsearch
+gdebi -n elasticsearch-2.1.1.deb
+rm elasticsearch-2.1.1.deb
+service elasticsearch start

 # -----------------------------------------------------------------------------
 # disable firewall

--- a/vm/config/environment
+++ b/vm/config/environment
 # Java
 JAVA_HOME=/usr/lib/jvm/java-7-openjdk-amd64/jre
-JAVA_TOOL_OPTIONS="-Xmx2048m -XX:MaxPermSize=1024m -Xms1024m"
-
-# Spark
-SPARK_HOME=/home/vagrant/spark
-SPARK_BIN=/home/vagrant/spark/bin
-
-# Mahout
-MAHOUT_HOME=/home/vagrant/mahout
-MAHOUT_LOCAL=true # for running standalone on your dev machine, unset MAHOUT_LOCAL for running on a cluster
-
-# Maven
-MAVEN_HOME=/home/vagrant/maven
-MAVEN_BIN=/home/vagrant/maven/bin
-
-PATH="/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/games:/usr/local/games:/home/vagrant/maven/bin"
\ No newline at end of file
+JAVA_TOOL_OPTIONS="-Xmx2048m -XX:MaxPermSize=1024m -Xms1024m"
\ No newline at end of file
--- a/vm/config/initd-tomcat
+++ b/vm/config/initd-tomcat
@@ -12,11 +12,11 @@
 PATH=/sbin:/bin:/usr/sbin:/usr/bin

 start() {
- /bin/su - tomcat -c /usr/share/tomcat/bin/startup.sh
+ /bin/su - vagrant -c /usr/share/tomcat/bin/startup.sh
 }

 stop() {
- /bin/su - tomcat -c /usr/share/tomcat/bin/shutdown.sh 
+ /bin/su - vagrant -c /usr/share/tomcat/bin/shutdown.sh 
 }

 case $1 in

--- a/vm/config/master-mongod.conf
+++ b/vm/config/master-mongod.conf
-# mongod.conf
-
-# for documentation of all options, see:
-#   http://docs.mongodb.org/manual/reference/configuration-options/
-
-# Where and how to store data.
-storage:
-  dbPath: /var/lib/mongodb
-  journal:
-    enabled: true
-#  engine:
-#  mmapv1:
-#  wiredTiger:
-
-# where to write logging data.
-systemLog:
-  destination: file
-  logAppend: true
-  path: /var/log/mongodb/mongod.log
-
-# network interfaces
-net:
-  port: 27017
-
-#processManagement:
-
-#security:
-
-#operationProfiling:
-
-#replication:
-
-#sharding:
-
-## Enterprise-Only Options:
-
-#auditLog:
-
-#snmp:
--- a/vm/config/spark-env.sh
+++ b/vm/config/spark-env.sh
-#!/usr/bin/env bash
-
-# This file is sourced when running various Spark programs.
-# Copy it as spark-env.sh and edit that to configure Spark for your site.
-
-# Options read when launching programs locally with
-# ./bin/run-example or ./bin/spark-submit
-# - HADOOP_CONF_DIR, to point Spark towards Hadoop configuration files
-# - SPARK_LOCAL_IP, to set the IP address Spark binds to on this node
-# - SPARK_PUBLIC_DNS, to set the public dns name of the driver program
-# - SPARK_CLASSPATH, default classpath entries to append
-
-# Options read by executors and drivers running inside the cluster
-# - SPARK_LOCAL_IP, to set the IP address Spark binds to on this node
-# - SPARK_PUBLIC_DNS, to set the public DNS name of the driver program
-# - SPARK_CLASSPATH, default classpath entries to append
-# - SPARK_LOCAL_DIRS, storage directories to use on this node for shuffle and RDD data
-# - MESOS_NATIVE_JAVA_LIBRARY, to point to your libmesos.so if you use Mesos
-
-# Options read in YARN client mode
-# - HADOOP_CONF_DIR, to point Spark towards Hadoop configuration files
-# - SPARK_EXECUTOR_INSTANCES, Number of workers to start (Default: 2)
-# - SPARK_EXECUTOR_CORES, Number of cores for the workers (Default: 1).
-# - SPARK_EXECUTOR_MEMORY, Memory per Worker (e.g. 1000M, 2G) (Default: 1G)
-# - SPARK_DRIVER_MEMORY, Memory for Master (e.g. 1000M, 2G) (Default: 1G)
-# - SPARK_YARN_APP_NAME, The name of your application (Default: Spark)
-# - SPARK_YARN_QUEUE, The hadoop queue to use for allocation requests (Default: ‘default’)
-# - SPARK_YARN_DIST_FILES, Comma separated list of files to be distributed with the job.
-# - SPARK_YARN_DIST_ARCHIVES, Comma separated list of archives to be distributed with the job.
-
-# Options for the daemons used in the standalone deploy mode
-# - SPARK_MASTER_IP, to bind the master to a different IP address or hostname
-# - SPARK_MASTER_PORT / SPARK_MASTER_WEBUI_PORT, to use non-default ports for the master
-# - SPARK_MASTER_OPTS, to set config properties only for the master (e.g. "-Dx=y")
-# - SPARK_WORKER_CORES, to set the number of cores to use on this machine
-# - SPARK_WORKER_MEMORY, to set how much total memory workers have to give executors (e.g. 1000m, 2g)
-# - SPARK_WORKER_PORT / SPARK_WORKER_WEBUI_PORT, to use non-default ports for the worker
-# - SPARK_WORKER_INSTANCES, to set the number of worker processes per node
-# - SPARK_WORKER_DIR, to set the working directory of worker processes
-# - SPARK_WORKER_OPTS, to set config properties only for the worker (e.g. "-Dx=y")
-# - SPARK_DAEMON_MEMORY, to allocate to the master, worker and history server themselves (default: 1g).
-# - SPARK_HISTORY_OPTS, to set config properties only for the history server (e.g. "-Dx=y")
-# - SPARK_SHUFFLE_OPTS, to set config properties only for the external shuffle service (e.g. "-Dx=y")
-# - SPARK_DAEMON_JAVA_OPTS, to set config properties for all daemons (e.g. "-Dx=y")
-# - SPARK_PUBLIC_DNS, to set the public dns name of the master or workers
-
-# Generic options for the daemons used in the standalone deploy mode
-# - SPARK_CONF_DIR      Alternate conf dir. (Default: ${SPARK_HOME}/conf)
-# - SPARK_LOG_DIR       Where log files are stored.  (Default: ${SPARK_HOME}/logs)
-# - SPARK_PID_DIR       Where the pid file is stored. (Default: /tmp)
-# - SPARK_IDENT_STRING  A string representing this instance of spark. (Default: $USER)
-# - SPARK_NICENESS      The scheduling priority for daemons. (Default: 0)
-
-SPARK_MASTER_PORT=7077
-SPARK_MASTER_WEBUI_PORT=8001
-SPARK_WORKER_WEBUI_PORT=8002
\ No newline at end of file
--- a/vm/config/tomcat-server.xml
+++ b/vm/config/tomcat-server.xml
-<?xml version='1.0' encoding='utf-8'?>
-<!--
-  Licensed to the Apache Software Foundation (ASF) under one or more
-  contributor license agreements.  See the NOTICE file distributed with
-  this work for additional information regarding copyright ownership.
-  The ASF licenses this file to You under the Apache License, Version 2.0
-  (the "License"); you may not use this file except in compliance with
-  the License.  You may obtain a copy of the License at
-
-      http://www.apache.org/licenses/LICENSE-2.0
-
-  Unless required by applicable law or agreed to in writing, software
-  distributed under the License is distributed on an "AS IS" BASIS,
-  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-  See the License for the specific language governing permissions and
-  limitations under the License.
-->
-<!-- Note:  A "Server" is not itself a "Container", so you may not
-     define subcomponents such as "Valves" at this level.
-     Documentation at /docs/config/server.html
- -->
-<Server port="8005" shutdown="SHUTDOWN">
-  <Listener className="org.apache.catalina.startup.VersionLoggerListener" />
-  <!-- Security listener. Documentation at /docs/config/listeners.html
-  <Listener className="org.apache.catalina.security.SecurityListener" />
-  -->
-  <!--APR library loader. Documentation at /docs/apr.html -->
-  <Listener className="org.apache.catalina.core.AprLifecycleListener" SSLEngine="on" />
-  <!-- Prevent memory leaks due to use of particular java/javax APIs-->
-  <Listener className="org.apache.catalina.core.JreMemoryLeakPreventionListener" />
-  <Listener className="org.apache.catalina.mbeans.GlobalResourcesLifecycleListener" />
-  <Listener className="org.apache.catalina.core.ThreadLocalLeakPreventionListener" />
-
-  <!-- Global JNDI resources
-       Documentation at /docs/jndi-resources-howto.html
-  -->
-  <GlobalNamingResources>
-    <!-- Editable user database that can also be used by
-         UserDatabaseRealm to authenticate users
-    -->
-    <Resource name="UserDatabase" auth="Container"
-              type="org.apache.catalina.UserDatabase"
-              description="User database that can be updated and saved"
-              factory="org.apache.catalina.users.MemoryUserDatabaseFactory"
-              pathname="conf/tomcat-users.xml" />
-  </GlobalNamingResources>
-
-  <!-- A "Service" is a collection of one or more "Connectors" that share
-       a single "Container" Note:  A "Service" is not itself a "Container",
-       so you may not define subcomponents such as "Valves" at this level.
-       Documentation at /docs/config/service.html
-   -->
-  <Service name="Catalina">
-
-    <!--The connectors can use a shared executor, you can define one or more named thread pools-->
-    <!--
-    <Executor name="tomcatThreadPool" namePrefix="catalina-exec-"
-        maxThreads="150" minSpareThreads="4"/>
-    -->
-
-
-    <!-- A "Connector" represents an endpoint by which requests are received
-         and responses are returned. Documentation at :
-         Java HTTP Connector: /docs/config/http.html (blocking & non-blocking)
-         Java AJP  Connector: /docs/config/ajp.html
-         APR (HTTP/AJP) Connector: /docs/apr.html
-         Define a non-SSL/TLS HTTP/1.1 Connector on port 8080
-    -->
-    <Connector port="8080" protocol="HTTP/1.1"
-               connectionTimeout="20000"
-               redirectPort="8443" />
-    <!-- A "Connector" using the shared thread pool-->
-    <!--
-    <Connector executor="tomcatThreadPool"
-               port="8080" protocol="HTTP/1.1"
-               connectionTimeout="20000"
-               redirectPort="8443" />
-    -->
-    <!-- Define a SSL/TLS HTTP/1.1 Connector on port 8443
-         This connector uses the NIO implementation that requires the JSSE
-         style configuration. When using the APR/native implementation, the
-         OpenSSL style configuration is required as described in the APR/native
-         documentation -->
-    <!--
-    <Connector port="8443" protocol="org.apache.coyote.http11.Http11NioProtocol"
-               maxThreads="150" SSLEnabled="true" scheme="https" secure="true"
-               clientAuth="false" sslProtocol="TLS" />
-    -->
-
-    <!-- Define an AJP 1.3 Connector on port 8009 -->
-    <Connector port="8009" protocol="AJP/1.3" redirectPort="8443" />
-
-
-    <!-- An Engine represents the entry point (within Catalina) that processes
-         every request.  The Engine implementation for Tomcat stand alone
-         analyzes the HTTP headers included with the request, and passes them
-         on to the appropriate Host (virtual host).
-         Documentation at /docs/config/engine.html -->
-
-    <!-- You should set jvmRoute to support load-balancing via AJP ie :
-    <Engine name="Catalina" defaultHost="localhost" jvmRoute="jvm1">
-    -->
-    <Engine name="Catalina" defaultHost="localhost">
-
-      <!--For clustering, please take a look at documentation at:
-          /docs/cluster-howto.html  (simple how to)
-          /docs/config/cluster.html (reference documentation) -->
-      <!--
-      <Cluster className="org.apache.catalina.ha.tcp.SimpleTcpCluster"/>
-      -->
-
-      <!-- Use the LockOutRealm to prevent attempts to guess user passwords
-           via a brute-force attack -->
-      <Realm className="org.apache.catalina.realm.LockOutRealm">
-        <!-- This Realm uses the UserDatabase configured in the global JNDI
-             resources under the key "UserDatabase".  Any edits
-             that are performed against this UserDatabase are immediately
-             available for use by the Realm.  -->
-        <Realm className="org.apache.catalina.realm.UserDatabaseRealm"
-               resourceName="UserDatabase"/>
-      </Realm>
-
-      <Host name="localhost"  appBase="webapps"
-            unpackWARs="true" autoDeploy="true">
-
-        <!-- SingleSignOn valve, share authentication between web applications
-             Documentation at: /docs/config/valve.html -->
-        <!--
-        <Valve className="org.apache.catalina.authenticator.SingleSignOn" />
-        -->
-
-        <!-- Access log processes all example.
-             Documentation at: /docs/config/valve.html
-             Note: The pattern used is equivalent to using pattern="common" -->
-        <Valve className="org.apache.catalina.valves.AccessLogValve" directory="logs"
-               prefix="localhost_access_log" suffix=".txt"
-               pattern="%h %l %u %t &quot;%r&quot; %s %b" />
-
-      </Host>
-    </Engine>
-  </Service>
-</Server>
--- a/vm/config/tomcat-users.xml
+++ b/vm/config/tomcat-users.xml
-<?xml version='1.0' encoding='utf-8'?>
-<!--
-  Licensed to the Apache Software Foundation (ASF) under one or more
-  contributor license agreements.  See the NOTICE file distributed with
-  this work for additional information regarding copyright ownership.
-  The ASF licenses this file to You under the Apache License, Version 2.0
-  (the "License"); you may not use this file except in compliance with
-  the License.  You may obtain a copy of the License at
-
-      http://www.apache.org/licenses/LICENSE-2.0
-
-  Unless required by applicable law or agreed to in writing, software
-  distributed under the License is distributed on an "AS IS" BASIS,
-  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-  See the License for the specific language governing permissions and
-  limitations under the License.
-->
-<tomcat-users xmlns="http://tomcat.apache.org/xml"
-              xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
-              xsi:schemaLocation="http://tomcat.apache.org/xml tomcat-users.xsd"
-              version="1.0">
-<!--
-  NOTE:  By default, no user is included in the "manager-gui" role required
-  to operate the "/manager/html" web application.  If you wish to use this app,
-  you must define such a user - the username and password are arbitrary.
-->
-<!--
-  NOTE:  The sample user and role entries below are wrapped in a comment
-  and thus are ignored when reading this file. Do not forget to remove
-  <!.. ..> that surrounds them.
-->
-<!--
-  <role rolename="tomcat"/>
-  <role rolename="role1"/>
-  <user username="tomcat" password="tomcat" roles="tomcat"/>
-  <user username="both" password="tomcat" roles="tomcat,role1"/>
-  <user username="role1" password="tomcat" roles="role1"/>
-->
-</tomcat-users>