Skip to content
Snippets Groups Projects
Commit 5c9e76f5 authored by Eike Cochu's avatar Eike Cochu
Browse files

updated vm environment variables 

rewrote provisioning script
added spark environment file
parent 70f36af1
No related branches found
No related tags found
No related merge requests found
Hide whitespace changes
Inline Side-by-side
Vagrantfile
+ 2
− 1
View file @ 5c9e76f5
......@@ -12,7 +12,8 @@ Vagrant.configure(2) do |config|
master.vm.network :private_network, ip: '192.168.10.10'
master.vm.network :forwarded_port, guest: 27017, host: 27017 # MongoDB
master.vm.network :forwarded_port, guest: 8081, host: 8081 # MongoDB Express
master.vm.network :forwarded_port, guest: 8080, host: 8888 # Spark MasterUI
master.vm.network :forwarded_port, guest: 8001, host: 8001 # Spark MasterUI
master.vm.network :forwarded_port, guest: 8002, host: 8002 # Spark WorkerUI
master.vm.network :forwarded_port, guest: 7077, host: 7077 # Spark
master.vm.network :forwarded_port, guest: 9200, host: 9200 # ElasticSearch REST API
master.vm.provision :shell, path: 'vm/bootstrap.sh'
......
ma-impl.sublime-workspace
+ 3
− 198
View file @ 5c9e76f5
......@@ -271,23 +271,6 @@
},
"buffers":
[
{
"contents": "renamed backend to tmbs-rest-backend\nmoved data dir to vm dir\nadded spark ports to vagrant config\nadded comments to vagrant ports\ninstalled elasticsearch in vm, added to bootstrap script",
"settings":
{
"buffer_size": 186,
"line_ending": "Unix",
"name": "renamed backend to tmbs-rest-backend"
}
},
{
"file": "Vagrantfile",
"settings":
{
"buffer_size": 797,
"line_ending": "Unix"
}
}
],
"build_system": "",
"build_system_choices":
......@@ -474,6 +457,9 @@
[
"/home/eike/Repositories/fu/ss15/ma/impl/vm/bootstrap.sh",
"/home/eike/Repositories/fu/ss15/ma/impl/Vagrantfile",
"/home/eike/Repositories/fu/ss15/ma/impl/vm/config/spark-env.sh",
"/home/eike/Repositories/fu/ss15/ma/impl/vm/rc.sh",
"/home/eike/Repositories/fu/ss15/ma/impl/vm/config/rc.local",
"/home/eike/Repositories/fu/ss15/ma/impl/vm/env.sh",
"/home/eike/Repositories/fu/ss15/ma/impl/ui/backend/pom.xml",
"/home/eike/Repositories/fu/ss15/ma/impl/vm/config/mongo-express.config.js",
......@@ -851,189 +837,8 @@
"groups":
[
{
"selected": 1,
"sheets":
[
{
"buffer": 0,
"semi_transient": false,
"settings":
{
"buffer_size": 186,
"regions":
{
},
"selection":
[
[
186,
186
]
],
"settings":
{
"BracketHighlighterBusy": false,
"auto_name": "renamed backend to tmbs-rest-backend",
"bh_regions":
[
"bh_square",
"bh_square_center",
"bh_square_open",
"bh_square_close",
"bh_square_content",
"bh_default",
"bh_default_center",
"bh_default_open",
"bh_default_close",
"bh_default_content",
"bh_single_quote",
"bh_single_quote_center",
"bh_single_quote_open",
"bh_single_quote_close",
"bh_single_quote_content",
"bh_round",
"bh_round_center",
"bh_round_open",
"bh_round_close",
"bh_round_content",
"bh_angle",
"bh_angle_center",
"bh_angle_open",
"bh_angle_close",
"bh_angle_content",
"bh_tag",
"bh_tag_center",
"bh_tag_open",
"bh_tag_close",
"bh_tag_content",
"bh_double_quote",
"bh_double_quote_center",
"bh_double_quote_open",
"bh_double_quote_close",
"bh_double_quote_content",
"bh_regex",
"bh_regex_center",
"bh_regex_open",
"bh_regex_close",
"bh_regex_content",
"bh_c_define",
"bh_c_define_center",
"bh_c_define_open",
"bh_c_define_close",
"bh_c_define_content",
"bh_curly",
"bh_curly_center",
"bh_curly_open",
"bh_curly_close",
"bh_curly_content",
"bh_unmatched",
"bh_unmatched_center",
"bh_unmatched_open",
"bh_unmatched_close",
"bh_unmatched_content"
],
"incomplete_sync": null,
"syntax": "Packages/Text/Plain text.tmLanguage"
},
"translation.x": 0.0,
"translation.y": 0.0,
"zoom_level": 1.0
},
"stack_index": 1,
"type": "text"
},
{
"buffer": 1,
"file": "Vagrantfile",
"semi_transient": true,
"settings":
{
"buffer_size": 797,
"regions":
{
},
"selection":
[
[
353,
353
]
],
"settings":
{
"BracketHighlighterBusy": false,
"bh_regions":
[
"bh_curly",
"bh_curly_center",
"bh_curly_open",
"bh_curly_close",
"bh_curly_content",
"bh_c_define",
"bh_c_define_center",
"bh_c_define_open",
"bh_c_define_close",
"bh_c_define_content",
"bh_angle",
"bh_angle_center",
"bh_angle_open",
"bh_angle_close",
"bh_angle_content",
"bh_round",
"bh_round_center",
"bh_round_open",
"bh_round_close",
"bh_round_content",
"bh_double_quote",
"bh_double_quote_center",
"bh_double_quote_open",
"bh_double_quote_close",
"bh_double_quote_content",
"bh_default",
"bh_default_center",
"bh_default_open",
"bh_default_close",
"bh_default_content",
"bh_regex",
"bh_regex_center",
"bh_regex_open",
"bh_regex_close",
"bh_regex_content",
"bh_square",
"bh_square_center",
"bh_square_open",
"bh_square_close",
"bh_square_content",
"bh_single_quote",
"bh_single_quote_center",
"bh_single_quote_open",
"bh_single_quote_close",
"bh_single_quote_content",
"bh_unmatched",
"bh_unmatched_center",
"bh_unmatched_open",
"bh_unmatched_close",
"bh_unmatched_content",
"bh_tag",
"bh_tag_center",
"bh_tag_open",
"bh_tag_close",
"bh_tag_content"
],
"incomplete_sync": null,
"remote_loading": false,
"synced": false,
"syntax": "Packages/Ruby/Ruby.sublime-syntax",
"tab_size": 2,
"translate_tabs_to_spaces": true
},
"translation.x": 0.0,
"translation.y": 0.0,
"zoom_level": 1.0
},
"stack_index": 0,
"type": "text"
}
]
}
],
......
vm/bootstrap.sh
+ 57
− 32
View file @ 5c9e76f5
#!/bin/sh
# -----------------------------------------------------------------------------
# add repos
apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv 7F0CEB10
echo "deb http://repo.mongodb.org/apt/ubuntu trusty/mongodb-org/3.0 multiverse" | tee /etc/apt/sources.list.d/mongodb-org-3.0.list
# -----------------------------------------------------------------------------
# upgrade system
apt-get update
apt-get upgrade -y
apt-get install git npm libkrb5-dev -y
# -----------------------------------------------------------------------------
# install packages
apt-get install git gdebi-core openjdk-7-jdk openjdk-7-jre -y
# -----------------------------------------------------------------------------
# install scala
wget http://www.scala-lang.org/files/archive/scala-2.11.7.deb
dpkg -i scala-2.11.7.deb
rm scala-2.11.7.deb
wget http://www.scala-lang.org/files/archive/scala-2.10.6.deb
gdebi -n scala-2.10.6.deb
rm scala-2.10.6.deb
# -----------------------------------------------------------------------------
# fix node setup in ubuntu
ln -s $(which nodejs) /bin/node
# install spark
wget http://mirror.netcologne.de/apache.org/spark/spark-1.5.2/spark-1.5.2-bin-hadoop2.6.tgz
tar zxf spark-1.5.2-bin-hadoop2.6.tgz
rm spark-1.5.2-bin-hadoop2.6.tgz
mv spark-1.5.2-bin-hadoop2.6 spark
ln -sf /vagrant/vm/config/spark-env.sh /home/vagrant/spark/conf/spark-env.sh
# -----------------------------------------------------------------------------
# install & start mongodb
apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv 7F0CEB10
echo "deb http://repo.mongodb.org/apt/ubuntu trusty/mongodb-org/3.0 multiverse" | tee /etc/apt/sources.list.d/mongodb-org-3.0.list
apt-get update
# install mongodb
apt-get install -y mongodb-org
# copy configuration
cp /vagrant/vm/config/master-mongod.conf /etc/mongod.conf
ln -sf /vagrant/vm/config/master-mongod.conf /etc/mongod.conf
# disable hugepages (https://docs.mongodb.org/manual/tutorial/transparent-huge-pages/)
cp /vagrant/vm/config/disable-transparent-hugepages /etc/init.d/
......@@ -33,38 +42,54 @@ update-rc.d disable-transparent-hugepages defaults
echo 'never' > /sys/kernel/mm/transparent_hugepage/enabled
echo 'never' > /sys/kernel/mm/transparent_hugepage/defrag
# install node, mongo-express
apt-get install npm libkrb5-dev
npm install -g mongo-express forever
cp /vagrant/vm/config/mongo-express.config.js /usr/local/lib/node_modules/mongo-express/config.js
# restart service
service mongod restart
# import mongodb data
mongoimport --db test --collection articles --file /vagrant/data/data.json --jsonArray
# start web ui
forever start $(which mongo-express)
mongoimport --db test --collection articles --file /vagrant/vm/data/data.json --jsonArray
# -----------------------------------------------------------------------------
# install spark
wget ftp://mirror.netcologne.de/apache.org/spark/spark-1.5.2/spark-1.5.2-bin-hadoop2.6.tgz
tar zxf spark-1.5.2-bin-hadoop2.6.tgz
rm spark-1.5.2-bin-hadoop2.6.tgz
mv spark-1.5.2-bin-hadoop2.6 spark
# install mongo express
apt-get install npm libkrb5-dev
ln -sf $(which nodejs) /bin/node
npm install -g mongo-express forever
ln -sf /vagrant/vm/config/mongo-express.config.js /usr/local/lib/node_modules/mongo-express/config.js
# -----------------------------------------------------------------------------
# install elasticsearch
wget -qO - https://packages.elastic.co/GPG-KEY-elasticsearch | apt-key add -
echo "deb http://packages.elastic.co/elasticsearch/2.x/debian stable main" | tee -a /etc/apt/sources.list.d/elasticsearch-2.x.list
apt-get install elasticsearch
update-rc.d elasticsearch defaults 95 10
# install maven
wget http://mirror.netcologne.de/apache.org/maven/maven-3/3.3.9/binaries/apache-maven-3.3.9-bin.tar.gz
tar zxf apache-maven-3.3.9-bin.tar.gz
mv apache-maven-3.3.9-bin maven
rm apache-maven-3.3.9-bin.tar.gz
# -----------------------------------------------------------------------------
# set environment
echo "source /vagrant/vm/env.sh" >> /home/vagrant/.bashrc
cat /vagrant/vm/config/environment >> /etc/environment
# -----------------------------------------------------------------------------
# install mahout
git clone https://github.com/apache/mahout.git mahout
cd mahout
mvn -DskipTests -X clean install
# -----------------------------------------------------------------------------
# disable firewall
ufw disable
\ No newline at end of file
ufw disable
# -----------------------------------------------------------------------------
# cleanup
apt-get autoremove -y
apt-get clean
dd if=/dev/zero of=/EMPTY bs=1M
rm -f /EMPTY
cat /dev/null > ~/.bash_history
history -c
echo ""
echo "--------------------------------------------------------------"
echo "--- Provisioning complete."
echo "--- Reload box to set environment"
echo "--- `vagrant reload`"
echo "--------------------------------------------------------------"
exit 0
\ No newline at end of file
vm/config/environment 0 → 100644
+ 17
− 0
View file @ 5c9e76f5
# Java
JAVA_HOME=/usr/lib/jvm/java-7-openjdk-amd64
JAVA_TOOL_OPTIONS="-Xmx2048m -XX:MaxPermSize=1024m -Xms1024m"
# Spark
SPARK_HOME=/home/vagrant/spark
SPARK_BIN=/home/vagrant/spark/bin
# Mahout
MAHOUT_HOME=/home/vagrant/mahout
MAHOUT_LOCAL=true # for running standalone on your dev machine, unset MAHOUT_LOCAL for running on a cluster
# Maven
MAVEN_HOME=/home/vagrant/maven
MAVEN_BIN=/home/vagrant/maven/bin
PATH="/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/games:/usr/local/games:/home/vagrant/maven/bin"
\ No newline at end of file
vm/config/spark-env.sh 0 → 100755
+ 56
− 0
View file @ 5c9e76f5
#!/usr/bin/env bash
# This file is sourced when running various Spark programs.
# Copy it as spark-env.sh and edit that to configure Spark for your site.
# Options read when launching programs locally with
# ./bin/run-example or ./bin/spark-submit
# - HADOOP_CONF_DIR, to point Spark towards Hadoop configuration files
# - SPARK_LOCAL_IP, to set the IP address Spark binds to on this node
# - SPARK_PUBLIC_DNS, to set the public dns name of the driver program
# - SPARK_CLASSPATH, default classpath entries to append
# Options read by executors and drivers running inside the cluster
# - SPARK_LOCAL_IP, to set the IP address Spark binds to on this node
# - SPARK_PUBLIC_DNS, to set the public DNS name of the driver program
# - SPARK_CLASSPATH, default classpath entries to append
# - SPARK_LOCAL_DIRS, storage directories to use on this node for shuffle and RDD data
# - MESOS_NATIVE_JAVA_LIBRARY, to point to your libmesos.so if you use Mesos
# Options read in YARN client mode
# - HADOOP_CONF_DIR, to point Spark towards Hadoop configuration files
# - SPARK_EXECUTOR_INSTANCES, Number of workers to start (Default: 2)
# - SPARK_EXECUTOR_CORES, Number of cores for the workers (Default: 1).
# - SPARK_EXECUTOR_MEMORY, Memory per Worker (e.g. 1000M, 2G) (Default: 1G)
# - SPARK_DRIVER_MEMORY, Memory for Master (e.g. 1000M, 2G) (Default: 1G)
# - SPARK_YARN_APP_NAME, The name of your application (Default: Spark)
# - SPARK_YARN_QUEUE, The hadoop queue to use for allocation requests (Default: ‘default’)
# - SPARK_YARN_DIST_FILES, Comma separated list of files to be distributed with the job.
# - SPARK_YARN_DIST_ARCHIVES, Comma separated list of archives to be distributed with the job.
# Options for the daemons used in the standalone deploy mode
# - SPARK_MASTER_IP, to bind the master to a different IP address or hostname
# - SPARK_MASTER_PORT / SPARK_MASTER_WEBUI_PORT, to use non-default ports for the master
# - SPARK_MASTER_OPTS, to set config properties only for the master (e.g. "-Dx=y")
# - SPARK_WORKER_CORES, to set the number of cores to use on this machine
# - SPARK_WORKER_MEMORY, to set how much total memory workers have to give executors (e.g. 1000m, 2g)
# - SPARK_WORKER_PORT / SPARK_WORKER_WEBUI_PORT, to use non-default ports for the worker
# - SPARK_WORKER_INSTANCES, to set the number of worker processes per node
# - SPARK_WORKER_DIR, to set the working directory of worker processes
# - SPARK_WORKER_OPTS, to set config properties only for the worker (e.g. "-Dx=y")
# - SPARK_DAEMON_MEMORY, to allocate to the master, worker and history server themselves (default: 1g).
# - SPARK_HISTORY_OPTS, to set config properties only for the history server (e.g. "-Dx=y")
# - SPARK_SHUFFLE_OPTS, to set config properties only for the external shuffle service (e.g. "-Dx=y")
# - SPARK_DAEMON_JAVA_OPTS, to set config properties for all daemons (e.g. "-Dx=y")
# - SPARK_PUBLIC_DNS, to set the public dns name of the master or workers
# Generic options for the daemons used in the standalone deploy mode
# - SPARK_CONF_DIR Alternate conf dir. (Default: ${SPARK_HOME}/conf)
# - SPARK_LOG_DIR Where log files are stored. (Default: ${SPARK_HOME}/logs)
# - SPARK_PID_DIR Where the pid file is stored. (Default: /tmp)
# - SPARK_IDENT_STRING A string representing this instance of spark. (Default: $USER)
# - SPARK_NICENESS The scheduling priority for daemons. (Default: 0)
SPARK_MASTER_PORT=7077
SPARK_MASTER_WEBUI_PORT=8001
SPARK_WORKER_WEBUI_PORT=8002
\ No newline at end of file
vm/env.sh deleted 100644 → 0
+ 0
− 2
View file @ 70f36af1
#!/bin/sh
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment