From 71a16b92e4059dc839115f7672b6ba1e829fc4a8 Mon Sep 17 00:00:00 2001
From: Eike Cochu <eike@cochu.com>
Date: Fri, 8 Jan 2016 14:25:18 +0100
Subject: [PATCH] updated vagrant box and provisioning
removed many errors and typos
removed spark, scala, maven, mahout
---
Vagrantfile | 4 -
vm/bootstrap.sh | 63 +++++-----------
vm/config/environment | 16 +---
vm/config/initd-tomcat | 4 +-
vm/config/master-mongod.conf | 39 ----------
vm/config/spark-env.sh | 56 --------------
vm/config/tomcat-server.xml | 142 -----------------------------------
vm/config/tomcat-users.xml | 39 ----------
8 files changed, 20 insertions(+), 343 deletions(-)
delete mode 100644 vm/config/master-mongod.conf
delete mode 100755 vm/config/spark-env.sh
delete mode 100644 vm/config/tomcat-server.xml
delete mode 100644 vm/config/tomcat-users.xml
diff --git a/Vagrantfile b/Vagrantfile
index 77a02e7b..a350bf2b 100644
--- a/Vagrantfile
+++ b/Vagrantfile
@@ -12,12 +12,8 @@ Vagrant.configure(2) do |config|
master.vm.network :private_network, ip: '192.168.10.10'
master.vm.network :forwarded_port, guest: 27017, host: 27017 # MongoDB
master.vm.network :forwarded_port, guest: 8081, host: 8081 # MongoDB Express
- master.vm.network :forwarded_port, guest: 8001, host: 8001 # Spark MasterUI
- master.vm.network :forwarded_port, guest: 8002, host: 8002 # Spark WorkerUI
- master.vm.network :forwarded_port, guest: 7077, host: 7077 # Spark
master.vm.network :forwarded_port, guest: 8080, host: 8000 # Tomcat
master.vm.network :forwarded_port, guest: 9200, host: 9200 # ElasticSearch REST API
- master.vm.provision :shell, path: 'vm/bootstrap.sh'
end
end
diff --git a/vm/bootstrap.sh b/vm/bootstrap.sh
index 0b238ae4..dac91121 100644
--- a/vm/bootstrap.sh
+++ b/vm/bootstrap.sh
@@ -6,8 +6,8 @@ CONFIG=$VMDIR/config
# -----------------------------------------------------------------------------
# add repos
-apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv 7F0CEB10
-echo "deb http://repo.mongodb.org/apt/ubuntu trusty/mongodb-org/3.0 multiverse" | tee /etc/apt/sources.list.d/mongodb-org-3.0.list
+apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv EA312927
+echo "deb http://repo.mongodb.org/apt/ubuntu trusty/mongodb-org/3.2 multiverse" | sudo tee /etc/apt/sources.list.d/mongodb-org-3.2.list
# -----------------------------------------------------------------------------
# upgrade system
@@ -18,27 +18,10 @@ apt-get upgrade -y
# install packages
apt-get install git gdebi-core openjdk-7-jdk openjdk-7-jre -y
-# -----------------------------------------------------------------------------
-# install scala
-wget http://www.scala-lang.org/files/archive/scala-2.10.6.deb
-gdebi -n scala-2.10.6.deb
-rm scala-2.10.6.deb
-
-# -----------------------------------------------------------------------------
-# install spark
-wget http://mirror.netcologne.de/apache.org/spark/spark-1.5.2/spark-1.5.2-bin-hadoop2.6.tgz
-tar zxf spark-1.5.2-bin-hadoop2.6.tgz
-rm spark-1.5.2-bin-hadoop2.6.tgz
-mv spark-1.5.2-bin-hadoop2.6 spark
-ln -sf $CONFIG/spark-env.sh /home/vagrant/spark/conf/spark-env.sh
-
# -----------------------------------------------------------------------------
# install mongodb
apt-get install -y mongodb-org
-# copy configuration
-ln -sf $CONFIG/master-mongod.conf /etc/mongod.conf
-
# disable hugepages (https://docs.mongodb.org/manual/tutorial/transparent-huge-pages/)
cp $CONFIG/disable-transparent-hugepages /etc/init.d/
chmod 755 /etc/init.d/disable-transparent-hugepages
@@ -49,9 +32,6 @@ echo 'never' > /sys/kernel/mm/transparent_hugepage/defrag
# restart service
service mongod restart
-# import mongodb data
-mongoimport --db test --collection articles --file $DATA/data.json --jsonArray
-
# -----------------------------------------------------------------------------
# install mongo express
apt-get install npm libkrb5-dev
@@ -61,16 +41,19 @@ ln -sf $CONFIG/mongo-express.config.js /usr/local/lib/node_modules/mongo-express
# -----------------------------------------------------------------------------
# install tomcat
-wget http://mirror.netcologne.de/apache.org/tomcat/tomcat-8/v8.0.30/bin/apache-tomcat-8.0.30.tar.gz
-tar zxf apache-tomcat-8.0.30.tar.gz
-mv apache-tomcat-8.0.30 /usr/share/tomcat8
-ln -s /usr/share/tomcat /usr/share/tomcat8
-# setup server config
-rm /usr/share/tomcat/conf/server.xml /usr/share/tomcat/conf/tomcat-users.xml
-ln -s $CONFIG/tomcat-server.xml /usr/share/tomcat/conf/server.xml
-ln -s $CONFIG/tomcat-users.xml /usr/share/tomcat/conf/tomcat-users.xml
+
+TOMCAT_MAJOR="8"
+TOMCAT_MINOR="0.30"
+TOMCAT_VERSION="$TOMCAT_MAJOR.$TOMCAT_MINOR"
+TOMCAT_ROOT=/usr/share/tomcat$TOMCAT_MAJOR
+
+wget http://mirror.netcologne.de/apache.org/tomcat/tomcat-$TOMCAT_MAJOR/v$TOMCAT_VERSION/bin/apache-tomcat-$TOMCAT_VERSION.tar.gz
+tar zxf apache-tomcat-$TOMCAT_VERSION.tar.gz
+rm apache-tomcat-$TOMCAT_VERSION.tar.gz
+mv apache-tomcat-$TOMCAT_VERSION $TOMCAT_ROOT
+ln -s $TOMCAT_ROOT /usr/share/tomcat
# set permissions
-chown -R vagrant:vagrant /usr/share/tomcat8
+chown -R vagrant:vagrant /usr/share/tomcat
chmod +x /usr/share/tomcat/bin/*.sh
# setup startup script
cp $CONFIG/initd-tomcat /etc/init.d/tomcat
@@ -82,24 +65,12 @@ ln -s $VMDIR/webapps /usr/share/tomcat/webapps
# start tomcat
service tomcat start
-# -----------------------------------------------------------------------------
-# install maven
-wget http://mirror.netcologne.de/apache.org/maven/maven-3/3.3.9/binaries/apache-maven-3.3.9-bin.tar.gz
-tar zxf apache-maven-3.3.9-bin.tar.gz
-mv apache-maven-3.3.9-bin maven
-rm apache-maven-3.3.9-bin.tar.gz
-
-# -----------------------------------------------------------------------------
-# install mahout
-git clone https://github.com/apache/mahout.git mahout
-mvn -DskipTests -X clean -f ./mahout install
-
# -----------------------------------------------------------------------------
# install elasticsearch
wget https://download.elasticsearch.org/elasticsearch/release/org/elasticsearch/distribution/deb/elasticsearch/2.1.1/elasticsearch-2.1.1.deb
-gdebi -n elasticsearch-2.1.0.deb
-rm elasticsearch-2.1.0.deb
-service start elasticsearch
+gdebi -n elasticsearch-2.1.1.deb
+rm elasticsearch-2.1.1.deb
+service elasticsearch start
# -----------------------------------------------------------------------------
# disable firewall
diff --git a/vm/config/environment b/vm/config/environment
index 4a081191..f7b4caf8 100644
--- a/vm/config/environment
+++ b/vm/config/environment
@@ -1,17 +1,3 @@
# Java
JAVA_HOME=/usr/lib/jvm/java-7-openjdk-amd64/jre
-JAVA_TOOL_OPTIONS="-Xmx2048m -XX:MaxPermSize=1024m -Xms1024m"
-
-# Spark
-SPARK_HOME=/home/vagrant/spark
-SPARK_BIN=/home/vagrant/spark/bin
-
-# Mahout
-MAHOUT_HOME=/home/vagrant/mahout
-MAHOUT_LOCAL=true # for running standalone on your dev machine, unset MAHOUT_LOCAL for running on a cluster
-
-# Maven
-MAVEN_HOME=/home/vagrant/maven
-MAVEN_BIN=/home/vagrant/maven/bin
-
-PATH="/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/games:/usr/local/games:/home/vagrant/maven/bin"
\ No newline at end of file
+JAVA_TOOL_OPTIONS="-Xmx2048m -XX:MaxPermSize=1024m -Xms1024m"
\ No newline at end of file
diff --git a/vm/config/initd-tomcat b/vm/config/initd-tomcat
index 35facc43..d7a2b738 100644
--- a/vm/config/initd-tomcat
+++ b/vm/config/initd-tomcat
@@ -12,11 +12,11 @@
PATH=/sbin:/bin:/usr/sbin:/usr/bin
start() {
- /bin/su - tomcat -c /usr/share/tomcat/bin/startup.sh
+ /bin/su - vagrant -c /usr/share/tomcat/bin/startup.sh
}
stop() {
- /bin/su - tomcat -c /usr/share/tomcat/bin/shutdown.sh
+ /bin/su - vagrant -c /usr/share/tomcat/bin/shutdown.sh
}
case $1 in
diff --git a/vm/config/master-mongod.conf b/vm/config/master-mongod.conf
deleted file mode 100644
index decdb1d9..00000000
--- a/vm/config/master-mongod.conf
+++ /dev/null
@@ -1,39 +0,0 @@
-# mongod.conf
-
-# for documentation of all options, see:
-# http://docs.mongodb.org/manual/reference/configuration-options/
-
-# Where and how to store data.
-storage:
- dbPath: /var/lib/mongodb
- journal:
- enabled: true
-# engine:
-# mmapv1:
-# wiredTiger:
-
-# where to write logging data.
-systemLog:
- destination: file
- logAppend: true
- path: /var/log/mongodb/mongod.log
-
-# network interfaces
-net:
- port: 27017
-
-#processManagement:
-
-#security:
-
-#operationProfiling:
-
-#replication:
-
-#sharding:
-
-## Enterprise-Only Options:
-
-#auditLog:
-
-#snmp:
diff --git a/vm/config/spark-env.sh b/vm/config/spark-env.sh
deleted file mode 100755
index 4cb916c1..00000000
--- a/vm/config/spark-env.sh
+++ /dev/null
@@ -1,56 +0,0 @@
-#!/usr/bin/env bash
-
-# This file is sourced when running various Spark programs.
-# Copy it as spark-env.sh and edit that to configure Spark for your site.
-
-# Options read when launching programs locally with
-# ./bin/run-example or ./bin/spark-submit
-# - HADOOP_CONF_DIR, to point Spark towards Hadoop configuration files
-# - SPARK_LOCAL_IP, to set the IP address Spark binds to on this node
-# - SPARK_PUBLIC_DNS, to set the public dns name of the driver program
-# - SPARK_CLASSPATH, default classpath entries to append
-
-# Options read by executors and drivers running inside the cluster
-# - SPARK_LOCAL_IP, to set the IP address Spark binds to on this node
-# - SPARK_PUBLIC_DNS, to set the public DNS name of the driver program
-# - SPARK_CLASSPATH, default classpath entries to append
-# - SPARK_LOCAL_DIRS, storage directories to use on this node for shuffle and RDD data
-# - MESOS_NATIVE_JAVA_LIBRARY, to point to your libmesos.so if you use Mesos
-
-# Options read in YARN client mode
-# - HADOOP_CONF_DIR, to point Spark towards Hadoop configuration files
-# - SPARK_EXECUTOR_INSTANCES, Number of workers to start (Default: 2)
-# - SPARK_EXECUTOR_CORES, Number of cores for the workers (Default: 1).
-# - SPARK_EXECUTOR_MEMORY, Memory per Worker (e.g. 1000M, 2G) (Default: 1G)
-# - SPARK_DRIVER_MEMORY, Memory for Master (e.g. 1000M, 2G) (Default: 1G)
-# - SPARK_YARN_APP_NAME, The name of your application (Default: Spark)
-# - SPARK_YARN_QUEUE, The hadoop queue to use for allocation requests (Default: ‘default’)
-# - SPARK_YARN_DIST_FILES, Comma separated list of files to be distributed with the job.
-# - SPARK_YARN_DIST_ARCHIVES, Comma separated list of archives to be distributed with the job.
-
-# Options for the daemons used in the standalone deploy mode
-# - SPARK_MASTER_IP, to bind the master to a different IP address or hostname
-# - SPARK_MASTER_PORT / SPARK_MASTER_WEBUI_PORT, to use non-default ports for the master
-# - SPARK_MASTER_OPTS, to set config properties only for the master (e.g. "-Dx=y")
-# - SPARK_WORKER_CORES, to set the number of cores to use on this machine
-# - SPARK_WORKER_MEMORY, to set how much total memory workers have to give executors (e.g. 1000m, 2g)
-# - SPARK_WORKER_PORT / SPARK_WORKER_WEBUI_PORT, to use non-default ports for the worker
-# - SPARK_WORKER_INSTANCES, to set the number of worker processes per node
-# - SPARK_WORKER_DIR, to set the working directory of worker processes
-# - SPARK_WORKER_OPTS, to set config properties only for the worker (e.g. "-Dx=y")
-# - SPARK_DAEMON_MEMORY, to allocate to the master, worker and history server themselves (default: 1g).
-# - SPARK_HISTORY_OPTS, to set config properties only for the history server (e.g. "-Dx=y")
-# - SPARK_SHUFFLE_OPTS, to set config properties only for the external shuffle service (e.g. "-Dx=y")
-# - SPARK_DAEMON_JAVA_OPTS, to set config properties for all daemons (e.g. "-Dx=y")
-# - SPARK_PUBLIC_DNS, to set the public dns name of the master or workers
-
-# Generic options for the daemons used in the standalone deploy mode
-# - SPARK_CONF_DIR Alternate conf dir. (Default: ${SPARK_HOME}/conf)
-# - SPARK_LOG_DIR Where log files are stored. (Default: ${SPARK_HOME}/logs)
-# - SPARK_PID_DIR Where the pid file is stored. (Default: /tmp)
-# - SPARK_IDENT_STRING A string representing this instance of spark. (Default: $USER)
-# - SPARK_NICENESS The scheduling priority for daemons. (Default: 0)
-
-SPARK_MASTER_PORT=7077
-SPARK_MASTER_WEBUI_PORT=8001
-SPARK_WORKER_WEBUI_PORT=8002
\ No newline at end of file
diff --git a/vm/config/tomcat-server.xml b/vm/config/tomcat-server.xml
deleted file mode 100644
index eb7dad74..00000000
--- a/vm/config/tomcat-server.xml
+++ /dev/null
@@ -1,142 +0,0 @@
-<?xml version='1.0' encoding='utf-8'?>
-<!--
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-<!-- Note: A "Server" is not itself a "Container", so you may not
- define subcomponents such as "Valves" at this level.
- Documentation at /docs/config/server.html
- -->
-<Server port="8005" shutdown="SHUTDOWN">
- <Listener className="org.apache.catalina.startup.VersionLoggerListener" />
- <!-- Security listener. Documentation at /docs/config/listeners.html
- <Listener className="org.apache.catalina.security.SecurityListener" />
- -->
- <!--APR library loader. Documentation at /docs/apr.html -->
- <Listener className="org.apache.catalina.core.AprLifecycleListener" SSLEngine="on" />
- <!-- Prevent memory leaks due to use of particular java/javax APIs-->
- <Listener className="org.apache.catalina.core.JreMemoryLeakPreventionListener" />
- <Listener className="org.apache.catalina.mbeans.GlobalResourcesLifecycleListener" />
- <Listener className="org.apache.catalina.core.ThreadLocalLeakPreventionListener" />
-
- <!-- Global JNDI resources
- Documentation at /docs/jndi-resources-howto.html
- -->
- <GlobalNamingResources>
- <!-- Editable user database that can also be used by
- UserDatabaseRealm to authenticate users
- -->
- <Resource name="UserDatabase" auth="Container"
- type="org.apache.catalina.UserDatabase"
- description="User database that can be updated and saved"
- factory="org.apache.catalina.users.MemoryUserDatabaseFactory"
- pathname="conf/tomcat-users.xml" />
- </GlobalNamingResources>
-
- <!-- A "Service" is a collection of one or more "Connectors" that share
- a single "Container" Note: A "Service" is not itself a "Container",
- so you may not define subcomponents such as "Valves" at this level.
- Documentation at /docs/config/service.html
- -->
- <Service name="Catalina">
-
- <!--The connectors can use a shared executor, you can define one or more named thread pools-->
- <!--
- <Executor name="tomcatThreadPool" namePrefix="catalina-exec-"
- maxThreads="150" minSpareThreads="4"/>
- -->
-
-
- <!-- A "Connector" represents an endpoint by which requests are received
- and responses are returned. Documentation at :
- Java HTTP Connector: /docs/config/http.html (blocking & non-blocking)
- Java AJP Connector: /docs/config/ajp.html
- APR (HTTP/AJP) Connector: /docs/apr.html
- Define a non-SSL/TLS HTTP/1.1 Connector on port 8080
- -->
- <Connector port="8080" protocol="HTTP/1.1"
- connectionTimeout="20000"
- redirectPort="8443" />
- <!-- A "Connector" using the shared thread pool-->
- <!--
- <Connector executor="tomcatThreadPool"
- port="8080" protocol="HTTP/1.1"
- connectionTimeout="20000"
- redirectPort="8443" />
- -->
- <!-- Define a SSL/TLS HTTP/1.1 Connector on port 8443
- This connector uses the NIO implementation that requires the JSSE
- style configuration. When using the APR/native implementation, the
- OpenSSL style configuration is required as described in the APR/native
- documentation -->
- <!--
- <Connector port="8443" protocol="org.apache.coyote.http11.Http11NioProtocol"
- maxThreads="150" SSLEnabled="true" scheme="https" secure="true"
- clientAuth="false" sslProtocol="TLS" />
- -->
-
- <!-- Define an AJP 1.3 Connector on port 8009 -->
- <Connector port="8009" protocol="AJP/1.3" redirectPort="8443" />
-
-
- <!-- An Engine represents the entry point (within Catalina) that processes
- every request. The Engine implementation for Tomcat stand alone
- analyzes the HTTP headers included with the request, and passes them
- on to the appropriate Host (virtual host).
- Documentation at /docs/config/engine.html -->
-
- <!-- You should set jvmRoute to support load-balancing via AJP ie :
- <Engine name="Catalina" defaultHost="localhost" jvmRoute="jvm1">
- -->
- <Engine name="Catalina" defaultHost="localhost">
-
- <!--For clustering, please take a look at documentation at:
- /docs/cluster-howto.html (simple how to)
- /docs/config/cluster.html (reference documentation) -->
- <!--
- <Cluster className="org.apache.catalina.ha.tcp.SimpleTcpCluster"/>
- -->
-
- <!-- Use the LockOutRealm to prevent attempts to guess user passwords
- via a brute-force attack -->
- <Realm className="org.apache.catalina.realm.LockOutRealm">
- <!-- This Realm uses the UserDatabase configured in the global JNDI
- resources under the key "UserDatabase". Any edits
- that are performed against this UserDatabase are immediately
- available for use by the Realm. -->
- <Realm className="org.apache.catalina.realm.UserDatabaseRealm"
- resourceName="UserDatabase"/>
- </Realm>
-
- <Host name="localhost" appBase="webapps"
- unpackWARs="true" autoDeploy="true">
-
- <!-- SingleSignOn valve, share authentication between web applications
- Documentation at: /docs/config/valve.html -->
- <!--
- <Valve className="org.apache.catalina.authenticator.SingleSignOn" />
- -->
-
- <!-- Access log processes all example.
- Documentation at: /docs/config/valve.html
- Note: The pattern used is equivalent to using pattern="common" -->
- <Valve className="org.apache.catalina.valves.AccessLogValve" directory="logs"
- prefix="localhost_access_log" suffix=".txt"
- pattern="%h %l %u %t "%r" %s %b" />
-
- </Host>
- </Engine>
- </Service>
-</Server>
diff --git a/vm/config/tomcat-users.xml b/vm/config/tomcat-users.xml
deleted file mode 100644
index 4dc60e63..00000000
--- a/vm/config/tomcat-users.xml
+++ /dev/null
@@ -1,39 +0,0 @@
-<?xml version='1.0' encoding='utf-8'?>
-<!--
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-<tomcat-users xmlns="http://tomcat.apache.org/xml"
- xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
- xsi:schemaLocation="http://tomcat.apache.org/xml tomcat-users.xsd"
- version="1.0">
-<!--
- NOTE: By default, no user is included in the "manager-gui" role required
- to operate the "/manager/html" web application. If you wish to use this app,
- you must define such a user - the username and password are arbitrary.
--->
-<!--
- NOTE: The sample user and role entries below are wrapped in a comment
- and thus are ignored when reading this file. Do not forget to remove
- <!.. ..> that surrounds them.
--->
-<!--
- <role rolename="tomcat"/>
- <role rolename="role1"/>
- <user username="tomcat" password="tomcat" roles="tomcat"/>
- <user username="both" password="tomcat" roles="tomcat,role1"/>
- <user username="role1" password="tomcat" roles="role1"/>
--->
-</tomcat-users>
--
GitLab