Welcome to Sahara!¶

(Optional)

# yum install openstack-sahara

# sahara-db-manage --config-file /etc/sahara/sahara.conf upgrade head

# systemctl start openstack-sahara-api
# systemctl start openstack-sahara-engine

# systemctl enable openstack-sahara-api
# systemctl enable openstack-sahara-engine

$ sudo apt-get install python-setuptools python-virtualenv python-dev

$ sudo yum install gcc python-setuptools python-virtualenv python-devel

$ sudo yum install gcc python-setuptools python-devel
$ sudo easy_install pip
$ sudo pip install virtualenv

$ virtualenv sahara-venv

$ sahara-venv/bin/pip install 'http://tarballs.openstack.org/sahara/sahara-master.tar.gz'

$ sahara-venv/bin/pip install 'http://tarballs.openstack.org/sahara/sahara-stable-release.tar.gz'

$ sahara-venv/bin/pip install 'http://tarballs.openstack.org/sahara/sahara-stable-mitaka.tar.gz'

$ mkdir sahara-venv/etc
$ cp sahara-venv/share/sahara/sahara.conf.sample-basic sahara-venv/etc/sahara.conf

...
[mysqld]
...
max_allowed_packet = 256M

$ sahara-venv/bin/sahara-db-manage --config-file sahara-venv/etc/sahara.conf upgrade head

# first terminal
$ sahara-venv/bin/sahara-api --config-file sahara-venv/etc/sahara.conf

# second terminal
$ sahara-venv/bin/sahara-engine --config-file sahara-venv/etc/sahara.conf

openstack service create --name sahara --description \
    "Sahara Data Processing" data-processing

openstack endpoint create --region RegionOne \
--publicurl http://10.0.0.2:8386/v1.1/%\(project_id\)s \
--adminurl http://10.0.0.2:8386/v1.1/%\(project_id\)s \
--internalurl http://10.0.0.2:8386/v1.1/%\(project_id\)s \
data-processing

[DEFAULT]
# Use m1.medium for {flavor_id} unless specified in another section
flavor_id = 2

[cdh-5-default-namenode]
# Use m1.large for {flavor_id} in the cdh-5-default-namenode template
flavor_id = 4

[cdh-530-default-namenode]
# Use m1.large for {flavor_id} in the cdh-530-default-namenode template
flavor_id = 4

$ sahara-templates --config-file /etc/sahara/sahara.conf --config-file myconfig update -t $PROJECT_ID

$ sahara-templates --help
$ sahara-templates update --help

$ sahara-venv/bin/python sahara-venv/bin/sahara-api --help
$ sahara-venv/bin/python sahara-venv/bin/sahara-engine --help

$ tox -e genconfig

connection=mysql://username:password@host:port/database

auth_uri=http://127.0.0.1:5000/v2.0/
identity_uri=http://127.0.0.1:35357/

use_neutron=true

[DEFAULT]
use_neutron=True
use_namespaces=True

[oslo_messaging_notifications]
enable = true

[oslo_messaging_notifications]
driver = messaging

rpc_backend = rabbit

rabbit_host=localhost
rabbit_port=5672
rabbit_hosts=$rabbit_host:$rabbit_port
rabbit_userid=guest
rabbit_password=guest
rabbit_virtual_host=/

{
    "default": ""
}

{
    "default": "",

    "data-processing:images:register": "role:admin",
    "data-processing:images:unregister": "role:admin",
    "data-processing:images:add_tags": "role:admin",
    "data-processing:images:remove_tags": "role:admin"
}

SAHARA_AUTO_IP_ALLOCATION_ENABLED = True

openstack service create --name sahara_local --description \
    "Sahara Data Processing (local installation)" \
    data_processing_local

openstack endpoint create --region RegionOne \
--publicurl http://127.0.0.1:8386/v1.1/%\(project_id\)s \
--adminurl http://127.0.0.1:8386/v1.1/%\(project_id\)s \
--internalurl http://127.0.0.1:8386/v1.1/%\(project_id\)s \
data_processing_local

# "type" of Sahara service registered in keystone
SAHARA_SERVICE = 'data_processing_local'

[DEFAULT]
proxy_command='ssh relay-machine-{tenant_id} nc {host} {port}'

[DEFAULT]
proxy_command='ip netns exec ns_for_{network_id} nc {host} {port}'

[DEFAULT]
# Use Designate for internal and external hostnames resolution:
use_designate=true
# IP addresses of Designate nameservers:
nameservers=1.1.1.1,2.2.2.2

[DEFAULT]
enable_data_locality=true

[DEFAULT]
compute_topology_file=/etc/sahara/compute.topology
swift_topology_file=/etc/sahara/swift.topology

compute1 /rack1
compute2 /rack2
compute3 /rack2

node1 /rack1
node2 /rack2
node3 /rack2

[DEFAULT]
periodic_coordinator_backend_url=kazoo://IP:PORT

pip install tooz[zookeeper]

[DEFAULT]
periodic_workers_number=2

[DEFAULT]
coordinator_heartbeat_interval=2

[DEFAULT]
hash_ring_replicas_count=100

[DEFAULT]
use_barbican_key_manager=true

[castellan]
barbican_api_endpoint=http://{barbican controller IP:PORT}/
barbican_api_version=v1

[DEFAULT]
os_region_name=RegionOne

[DEFAULT]
use_rootwrap=true

sahara ALL = (root) NOPASSWD: /usr/bin/sahara-rootwrap /etc/sahara/rootwrap.conf *

[Filters]
ip: IpNetnsExecFilter, ip, root
nc: CommandFilter, nc, root
kill: CommandFilter, kill, root

[DEFAULT]
rootwrap_command='sudo sahara-rootwrap /etc/sahara/rootwrap.conf'

[DEFAULT]
use_domain_for_proxy_users=true
proxy_user_domain_name=sahara_proxy
proxy_user_role_names=Member,SwiftUser

os_privileged_user_name =
os_privileged_user_password =
os_privileged_user_tenant =

cluster_configs: {
    "general": {
        "URL of NTP server": "your_server.net",
    }
}

"cluster_configs": {
    "general": {
        "Enable NTP service": false,
    }
}

[cors]
allowed_origin=https://we.example.com:443
max_age=3600
allow_credentials=true

[cors.additional_domain_1]
allowed_origin=https://additional_domain_1.example.com:443

[cors.additional_domain_2]
allowed_origin=https://additional_domain_2.example.com:443

[DEFAULT]
cleanup_time_for_incomplete_clusters = 3

[quotas]
quota_security_group = 1000
quota_security_group_rule = 10000

openstack quota set --secgroups 1000 --secgroup-rules 10000 $PROJECT_ID

cat update_configs.json
{
    "plugin_labels": {
        "enabled": {
            "status": true
        }
    }
}
openstack dataprocessing plugin update vanilla update_configs.json

cat update_configs.json
{
    "version_labels": {
        "2.7.1": {
            "enabled": {
                "status": true
            }
        }
    }
}
openstack dataprocessing plugin update vanilla update_configs.json

openstack dataprocessing plugin show vanilla

"cluster_configs": {
  "Enable Kerberos Security": true,
}

"cluster_configs": {
  "Enable Kerberos Security": true,
}

[
    {
        "name": "Example Class",
        "description": "Indicates which example job class should be used.",
        "mapping_type": "args",
        "location": "0",
        "value_type": "string",
        "required": false,
        "default": "teragen"
    },
]

{
    "Rows": "1000000",
    "Mapper Count": "3",
    "Output Path": "hdfs://mycluster:8020/user/myuser/teragen-output"
}

{
    "job_configs": {
        "configs":
            {
                "mapred.map.tasks": "3"
            },
        "args":
            [
                "teragen",
                "1000000",
                "hdfs://mycluster:8020/user/myuser/teragen-output"
            ]
    }
}

GET /v1.1/{project_id}/clusters.json

GET /v1.1/{project_id}/clusters
Accept: application/json

GET /v1.1/{project_id}/clusters?limit=15&marker=d62ad147-5c10-418c-a21a-3a6597044f29

GET /v1.1/{project_id}/clusters?sort_by=name

GET /v1.1/{project_id}/clusters?sort_by=-created_at

HTTP/1.1 400 BAD REQUEST
Content-type: application/json
Content-length: 126

{
    "error_name": "CLUSTER_NAME_ALREADY_EXISTS",
    "error_message": "Cluster with name 'test-cluster' already exists",
    "error_code": 400
}

<property>
    <name>${name} + ${config}</name>
    <value>${value}</value>
    <description>${not mandatory description}</description>
</property>

$ hadoop distcp -D fs.swift.service.sahara.username=admin \
 -D fs.swift.service.sahara.password=swordfish \
 swift://integration.sahara/temp swift://integration.sahara/temp1

tox -e venv -- sahara-image-create -u

tox -e venv -- sahara-image-create -p vanilla -v 2.7.1 -i centos7

qemu-img convert -O vmdk <original_image>.qcow2 <converted_image>.vmdk

tox -e venv -- sahara-image-create -p cloudera -v 5.5

tox -e venv -- sahara-image-create -p cloudera -i ubuntu -v 5.5

export DIB_CDH_MINOR_VERSION=5.7.1

qemu-img convert -O vmdk <original_image>.qcow2 <converted_image>.vmdk

$ tox -e pep8

$ tox -e pylint

$ pwd
/home/me/sahara/sahara/plugins/vanilla/v2_7_1/resources

$ ls
core-default.xml     hdfs-default.xml    oozie-default.xml   README.rst
create_oozie_db.sql  mapred-default.xml  post_conf.template  yarn-default.xml

$ tox -e py27

$ tox -e docs

$ SPHINX_DEBUG=1 tox -e docs

$ SPHINX_DEBUG=1 .tox/docs/bin/python setup.py build_sphinx

from sahara.service.castellan import utils as key_manager

password = 'SooperSecretPassword'
identifier = key_manager.store_secret(password)

from sahara.service.castellan import utils as key_manager

password = key_manager.get_secret(identifier)

from sahara.service.castellan import utils as key_manager

key_manager.delete_secret(identifier)

# we actually need pip, which is part of python package
$ brew install python mysql postgresql rabbitmq
$ pip install virtualenv tox

$ sudo apt-get update
$ sudo apt-get install git-core python-dev python-virtualenv gcc libpq-dev libmysqlclient-dev python-pip rabbitmq-server
$ sudo pip install tox

$ sudo yum install git-core python-devel python-virtualenv gcc python-pip mariadb-devel postgresql-devel erlang
$ sudo pip install tox
$ sudo wget http://www.rabbitmq.com/releases/rabbitmq-server/v3.2.2/rabbitmq-server-3.2.2-1.noarch.rpm
$ sudo rpm --import http://www.rabbitmq.com/rabbitmq-signing-key-public.asc
$ sudo yum install rabbitmq-server-3.2.2-1.noarch.rpm

$ sudo zypper in gcc git libmysqlclient-devel postgresql-devel python-devel python-pip python-tox python-virtualenv

$ git clone git://github.com/openstack/sahara.git
$ cd sahara

tox -e genconfig

$ cp ./etc/sahara/sahara.conf.sample ./etc/sahara/sahara.conf

$ tox -e venv -- sahara-db-manage --config-file etc/sahara/sahara.conf upgrade head

$ tox -e venv -- sahara-api --config-file etc/sahara/sahara.conf --debug
$ tox -e venv -- sahara-engine --config-file etc/sahara/sahara.conf --debug

# Enable sahara-dashboard
enable_plugin sahara-dashboard git://git.openstack.org/openstack/sahara-dashboard

$ openstack endpoint list

$ openstack service list

$ sudo apt-get update
$ sudo apt-get install git-core python-dev gcc python-setuptools \
          python-virtualenv node-less libssl-dev libffi-dev libxslt-dev

$ sudo apt-get install nodejs-legacy

$ git clone https://git.openstack.org/cgit/openstack/horizon/ {HORIZON_DIR}

$ python {HORIZON_DIR}/tools/install_venv.py

$ cp {HORIZON_DIR}/openstack_dashboard/local/local_settings.py.example
          {HORIZON_DIR}/openstack_dashboard/local/local_settings.py

OPENSTACK_HOST = "ip of your controller"

SAHARA_AUTO_IP_ALLOCATION_ENABLED = True

$ git clone https://git.openstack.org/cgit/openstack/sahara-dashboard/ \
    {SAHARA_DASHBOARD_DIR}

$ cp -a {SAHARA_DASHBOARD_DIR}/sahara_dashboard/enabled/* {HORIZON_DIR}/openstack_dashboard/local/enabled/

$ source {HORIZON_DIR}/.venv/bin/activate
$ pip install -e {SAHARA_DASHBOARD_DIR}

$ source {HORIZON_DIR}/.venv/bin/activate
$ python {HORIZON_DIR}/manage.py runserver 0.0.0.0:8080

$ source {HORIZON_DIR}/.venv/bin/activate
$ python {HORIZON_DIR}/manage.py runserver --insecure 0.0.0.0:8080

[global]
download-cache = /home/<username>/.pip/cache
index-url = <mirror url>

#!/bin/sh
# Run fast checks (PEP8 style check and PyFlakes fast static analysis)
tox -epep8

tox -e pylint

$ sudo apt-get install git-core
$ git clone https://git.openstack.org/openstack-dev/devstack.git

[[local|localrc]]
ADMIN_PASSWORD=nova
MYSQL_PASSWORD=nova
RABBIT_PASSWORD=nova
SERVICE_PASSWORD=$ADMIN_PASSWORD
SERVICE_TOKEN=nova

# Enable Swift
enable_service s-proxy s-object s-container s-account

SWIFT_HASH=66a3d6b56c1f479c8b4e70ab5c2000f5
SWIFT_REPLICAS=1
SWIFT_DATA_DIR=$DEST/data

# Force checkout prerequisites
# FORCE_PREREQ=1

# keystone is now configured by default to use PKI as the token format
# which produces huge tokens.
# set UUID as keystone token format which is much shorter and easier to
# work with.
KEYSTONE_TOKEN_FORMAT=UUID

# Change the FLOATING_RANGE to whatever IPs VM is working in.
# In NAT mode it is the subnet VMware Fusion provides, in bridged mode
# it is your local network. But only use the top end of the network by
# using a /27 and starting at the 224 octet.
FLOATING_RANGE=192.168.55.224/27

# Enable logging
SCREEN_LOGDIR=$DEST/logs/screen

# Set ``OFFLINE`` to ``True`` to configure ``stack.sh`` to run cleanly
# without Internet access. ``stack.sh`` must have been previously run
# with Internet access to install prerequisites and fetch repositories.
# OFFLINE=True

# Enable sahara
enable_plugin sahara git://git.openstack.org/openstack/sahara

enable_plugin sahara git://git.openstack.org/openstack/sahara <some_git_refspec>

enable_plugin ceilometer git://git.openstack.org/openstack/ceilometer

$ ./stack.sh

subnet 192.168.55.0 netmask 255.255.255.0 {
        range 192.168.55.128 192.168.55.254;

host VM_HOSTNAME {
        hardware ethernet VM_MAC_ADDRESS;
        fixed-address 192.168.55.20;
}

$ sudo /Applications/VMware\ Fusion.app/Contents/Library/vmnet-cli --stop
$ sudo /Applications/VMware\ Fusion.app/Contents/Library/vmnet-cli --start

$ export OS_AUTH_URL=http://127.0.0.1:5000/v2.0/
$ export OS_PROJECT_NAME=admin
$ export OS_USERNAME=admin
$ export OS_PASSWORD=nova

$ ssh user@hostname
$ wget http://sahara-files.mirantis.com/images/upstream/<openstack_release>/<sahara_image>.qcow2

$ openstack image create sahara-vanilla-latest-ubuntu --disk-format qcow2 \
    --container-format bare --file sahara-vanilla-latest-ubuntu.qcow2
+------------------+--------------------------------------+
| Field            | Value                                |
+------------------+--------------------------------------+
| checksum         | 3da49911332fc46db0c5fb7c197e3a77     |
| container_format | bare                                 |
| created_at       | 2016-02-29T10:15:04.000000           |
| deleted          | False                                |
| deleted_at       | None                                 |
| disk_format      | qcow2                                |
| id               | 71b9eeac-c904-4170-866a-1f833ea614f3 |
| is_public        | False                                |
| min_disk         | 0                                    |
| min_ram          | 0                                    |
| name             | sahara-vanilla-latest-ubuntu         |
| owner            | 057d23cddb864759bfa61d730d444b1f     |
| properties       |                                      |
| protected        | False                                |
| size             | 1181876224                           |
| status           | active                               |
| updated_at       | 2016-02-29T10:15:41.000000           |
| virtual_size     | None                                 |
+------------------+--------------------------------------+

$ openstack image list --property name=sahara-vanilla-latest-ubuntu
+--------------------------------------+------------------------------+
| ID                                   | Name                         |
+--------------------------------------+------------------------------+
| 71b9eeac-c904-4170-866a-1f833ea614f3 | sahara-vanilla-latest-ubuntu |
+--------------------------------------+------------------------------+

$ openstack dataprocessing image register sahara-vanilla-latest-ubuntu \
    --username ubuntu

$ openstack dataprocessing image tags add sahara-vanilla-latest-ubuntu \
    --tags vanilla <plugin_version>
+-------------+--------------------------------------+
| Field       | Value                                |
+-------------+--------------------------------------+
| Description | None                                 |
| Id          | 71b9eeac-c904-4170-866a-1f833ea614f3 |
| Name        | sahara-vanilla-latest-ubuntu         |
| Status      | ACTIVE                               |
| Tags        | <plugin_version>, vanilla            |
| Username    | ubuntu                               |
+-------------+--------------------------------------+

$ openstack dataprocessing plugin list

$ openstack dataprocessing plugin show vanilla --plugin-version <plugin_version>
+---------------------+-----------------------------------------------------------------------------------------------------------------------+
| Field               | Value                                                                                                                 |
+---------------------+-----------------------------------------------------------------------------------------------------------------------+
| Description         | The Apache Vanilla plugin provides the ability to launch upstream Vanilla Apache Hadoop cluster without any           |
|                     | management consoles. It can also deploy the Oozie component.                                                          |
| Name                | vanilla                                                                                                               |
| Required image tags | <plugin_version>, vanilla                                                                                             |
| Title               | Vanilla Apache Hadoop                                                                                                 |
|                     |                                                                                                                       |
| Service:            | Available processes:                                                                                                  |
|                     |                                                                                                                       |
| HDFS                | datanode, namenode, secondarynamenode                                                                                 |
| Hadoop              |                                                                                                                       |
| Hive                | hiveserver                                                                                                            |
| JobFlow             | oozie                                                                                                                 |
| Spark               | spark history server                                                                                                  |
| MapReduce           | historyserver                                                                                                         |
| YARN                | nodemanager, resourcemanager                                                                                          |
+---------------------+-----------------------------------------------------------------------------------------------------------------------+

$ openstack dataprocessing node group template create \
    --name vanilla-default-master --plugin vanilla \
    --plugin-version <plugin_version> --processes namenode resourcemanager \
    --flavor 2 --auto-security-group --floating-ip-pool <pool-id>
+---------------------+--------------------------------------+
| Field               | Value                                |
+---------------------+--------------------------------------+
| Auto security group | True                                 |
| Availability zone   | None                                 |
| Flavor id           | 2                                    |
| Floating ip pool    | dbd8d1aa-6e8e-4a35-a77b-966c901464d5 |
| Id                  | 0f066e14-9a73-4379-bbb4-9d9347633e31 |
| Is default          | False                                |
| Is protected        | False                                |
| Is proxy gateway    | False                                |
| Is public           | False                                |
| Name                | vanilla-default-master               |
| Node processes      | namenode, resourcemanager            |
| Plugin name         | vanilla                              |
| Security groups     | None                                 |
| Use autoconfig      | False                                |
| Version             | <plugin_version>                     |
| Volumes per node    | 0                                    |
+---------------------+--------------------------------------+

$ openstack dataprocessing node group template create \
    --name vanilla-default-worker --plugin vanilla \
    --plugin-version <plugin_version> --processes datanode nodemanager \
    --flavor 2 --auto-security-group --floating-ip-pool <pool-id>
+---------------------+--------------------------------------+
| Field               | Value                                |
+---------------------+--------------------------------------+
| Auto security group | True                                 |
| Availability zone   | None                                 |
| Flavor id           | 2                                    |
| Floating ip pool    | dbd8d1aa-6e8e-4a35-a77b-966c901464d5 |
| Id                  | 6546bf44-0590-4539-bfcb-99f8e2c11efc |
| Is default          | False                                |
| Is protected        | False                                |
| Is proxy gateway    | False                                |
| Is public           | False                                |
| Name                | vanilla-default-worker               |
| Node processes      | datanode, nodemanager                |
| Plugin name         | vanilla                              |
| Security groups     | None                                 |
| Use autoconfig      | False                                |
| Version             | <plugin_version>                     |
| Volumes per node    | 0                                    |
+---------------------+--------------------------------------+

{
    "plugin_name": "vanilla",
    "hadoop_version": "<plugin_version>",
    "node_processes": [
        "namenode",
        "resourcemanager"
    ],
    "name": "vanilla-default-master",
    "floating_ip_pool": "<floating_ip_pool_id>",
    "flavor_id": "2",
    "auto_security_group": true
}

{
    "plugin_name": "vanilla",
    "hadoop_version": "<plugin_version>",
    "node_processes": [
        "nodemanager",
        "datanode"
    ],
    "name": "vanilla-default-worker",
    "floating_ip_pool": "<floating_ip_pool_id>",
    "flavor_id": "2",
    "auto_security_group": true
}

$ openstack dataprocessing node group template create \
    --json my_master_template_create.json
$ openstack dataprocessing node group template create \
    --json my_worker_template_create.json

$ openstack dataprocessing node group template list --name vanilla-default
+------------------------+--------------------------------------+-------------+--------------------+
| Name                   | Id                                   | Plugin name | Version            |
+------------------------+--------------------------------------+-------------+--------------------+
| vanilla-default-master | 0f066e14-9a73-4379-bbb4-9d9347633e31 | vanilla     | <plugin_version>   |
| vanilla-default-worker | 6546bf44-0590-4539-bfcb-99f8e2c11efc | vanilla     | <plugin_version>   |
+------------------------+--------------------------------------+-------------+--------------------+

$ openstack dataprocessing cluster template create \
    --name vanilla-default-cluster \
    --node-groups vanilla-default-master:1 vanilla-default-worker:3

+----------------+----------------------------------------------------+
| Field          | Value                                              |
+----------------+----------------------------------------------------+
| Anti affinity  |                                                    |
| Description    | None                                               |
| Id             | 9d871ebd-88a9-40af-ae3e-d8c8f292401c               |
| Is default     | False                                              |
| Is protected   | False                                              |
| Is public      | False                                              |
| Name           | vanilla-default-cluster                            |
| Node groups    | vanilla-default-master:1, vanilla-default-worker:3 |
| Plugin name    | vanilla                                            |
| Use autoconfig | False                                              |
| Version        | <plugin_version>                                   |
+----------------+----------------------------------------------------+

{
    "plugin_name": "vanilla",
    "hadoop_version": "<plugin_version>",
    "node_groups": [
        {
            "name": "worker",
            "count": 3,
            "node_group_template_id": "6546bf44-0590-4539-bfcb-99f8e2c11efc"
        },
        {
            "name": "master",
            "count": 1,
            "node_group_template_id": "0f066e14-9a73-4379-bbb4-9d9347633e31"
        }
    ],
    "name": "vanilla-default-cluster",
    "cluster_configs": {}
}

$ openstack dataprocessing cluster template create --json my_cluster_template_create.json

$ openstack dataprocessing cluster template list --name vanilla-default
+-------------------------+--------------------------------------+-------------+--------------------+
| Name                    | Id                                   | Plugin name | Version            |
+-------------------------+--------------------------------------+-------------+--------------------+
| vanilla-default-cluster | 9d871ebd-88a9-40af-ae3e-d8c8f292401c | vanilla     | <plugin_version>   |
+-------------------------+--------------------------------------+-------------+--------------------+

$ openstack dataprocessing cluster create --name my-cluster-1 \
    --cluster-template vanilla-default-cluster --user-keypair my_stack \
    --neutron-network private --image sahara-vanilla-latest-ubuntu

+----------------------------+----------------------------------------------------+
| Field                      | Value                                              |
+----------------------------+----------------------------------------------------+
| Anti affinity              |                                                    |
| Cluster template id        | 9d871ebd-88a9-40af-ae3e-d8c8f292401c               |
| Description                |                                                    |
| Id                         | 1f0dc6f7-6600-495f-8f3a-8ac08cdb3afc               |
| Image                      | 71b9eeac-c904-4170-866a-1f833ea614f3               |
| Is protected               | False                                              |
| Is public                  | False                                              |
| Is transient               | False                                              |
| Name                       | my-cluster-1                                       |
| Neutron management network | fabe9dae-6fbd-47ca-9eb1-1543de325efc               |
| Node groups                | vanilla-default-master:1, vanilla-default-worker:3 |
| Plugin name                | vanilla                                            |
| Status                     | Validating                                         |
| Use autoconfig             | False                                              |
| User keypair id            | my_stack                                           |
| Version                    | <plugin_version>                                   |
+----------------------------+----------------------------------------------------+

{
    "name": "my-cluster-1",
    "plugin_name": "vanilla",
    "hadoop_version": "<plugin_version>",
    "cluster_template_id" : "9d871ebd-88a9-40af-ae3e-d8c8f292401c",
    "user_keypair_id": "my_stack",
    "default_image_id": "71b9eeac-c904-4170-866a-1f833ea614f3",
    "neutron_management_network": "fabe9dae-6fbd-47ca-9eb1-1543de325efc"
}

$ openstack keypair create my_stack --public-key $PATH_TO_PUBLIC_KEY

$ openstack network list

$ openstack dataprocessing cluster create --json my_cluster_create.json

$ openstack dataprocessing cluster show my-cluster-1 -c Status
+--------+--------+
| Field  | Value  |
+--------+--------+
| Status | Active |
+--------+--------+

$ ssh -i my_stack.pem ubuntu@<namenode_ip>

$ sudo su hadoop

$ cd /opt/hadoop-<plugin_version>/share/hadoop/mapreduce
$ /opt/hadoop-<plugin_version>/bin/hadoop jar hadoop-mapreduce-examples-<plugin_version>.jar pi 10 100

(openstack) dataprocessing job binary create --url "swift://integration.sahara/hive.sql" \
  --username username --password password --description "My first job binary" hive-binary

$ openstack dataprocessing data source create --type swift --username admin --password admin \
   --url "swift://integration.sahara/input.txt" input

$ openstack dataprocessing data source create --type swift --username admin --password admin \
   --url "swift://integration.sahara/output.txt" input

$ openstack dataprocessing data source create --type hdfs --url "hdfs://tmp/input.txt" input

$ openstack dataprocessing data source create --type hdfs --url "hdfs://tmp/output.txt" output

$ openstack dataprocessing job template create --type Hive \
   --name hive-job-template --main hive-binary

$ openstack dataprocessing job execute --input input --output output \
  --job-template hive-job-template --cluster my-first-cluster

$ openstack dataprocessing job show <id_of_your_job>

$ tar -xzvf oozie-4.0.1.tar.gz

$ {oozie_dir}/bin/mkdistro.sh -DskipTests

$ find . -name pom.xml | xargs sed -ri 's/2.3.0/2.x.x/'

$ {oozie_dir}/bin/mkdistro.sh assembly:single -P hadoop-2 -D javaVersion=1.7 -D targetJavaVersion=1.7 -D skipTests

$ sahara-db-manage --config-file /path/to/sahara.conf revision -m "description of revision"

$ sahara-db-manage --config-file /path/to/sahara.conf revision -m "description of revision" --autogenerate

# revision identifiers, used by Alembic.
revision = '507eb70202af'
down_revision = '006'

# revision identifiers, used by Alembic.
revision = '007'
down_revision = '006'

$ mv 507eb70202af_my_new_revision.py 007_my_new_revision.py

$ sahara-db-manage --config-file /path/to/sahara.conf upgrade head

$ sahara-db-manage --config-file /path/to/sahara.conf current

$ sahara-db-manage --config-file /path/to/sahara.conf upgrade head --sql

$ sahara-db-manage --config-file /path/to/sahara.conf upgrade <start version>:<end version> --sql

$ sahara-db-manage --config-file /path/to/sahara.conf upgrade --delta <# of revs>

$ sahara-db-manage --config-file /path/to/sahara.conf revision -m "description of revision" --autogenerate

$ sahara-db-manage --config-file /path/to/sahara.conf revision -m "description of revision"

$ sahara-db-manage --config-file /path/to/sahara.conf stamp <revision>

$ sahara-db-manage --config-file /path/to/sahara.conf check_migration

$ sahara-db-manage --config-file /path/to/sahara.conf history

LOG.warning(_LW("Incorrect path: {path}").format(path=path))

[app:sahara_apiv2]
paste.app_factory = sahara.api.middleware.sahara_middleware:RouterV2.factory

[filter:auth_validator_v2]
paste.filter_factory = sahara.api.middleware.auth_valid:AuthValidatorV2.factory

[pipeline:sahara]
pipeline = cors request_id acl auth_validator_v2 sahara_api

[composite:sahara_api]
use = egg:Paste#urlmap
/: sahara_apiv2

GET /v1.1/12345678-1234-1234-1234-123456789ABC/clusters
X-Auth-Token: {valid auth token}

GET /v2/clusters
X-Auth-Token: {valid auth token}
OpenStack-Project-ID: 12345678-1234-1234-1234-123456789ABC

$ httpie http://{sahara service ip:port}/v2/clusters \
  X-Auth-Token:{valid auth token} \
  OpenStack-Project-ID:12345678-1234-1234-1234-123456789ABC

$ qemu-img convert -O raw image-converted.qcow image-converted-from-qcow2.raw

$ ironic node-create -d pxe_ipmitool \
$ -i ipmi_address=$IP_ADDRESS \
$ -i ipmi_username=$USERNAME \
$ -i ipmi_password=$PASSWORD \
$ -i pxe_deploy_kernel=$deploy.kernel.id \
$ -i pxe_deploy_ramdisk=$deploy.ramfs.id

$ ironic port-create -n $NODE_ID -a "$MAC_eth1"

$ ironic node-update $NODE_ID add properties/cpus=$CPU \
$ properties/memory_mb=$RAM properties/local_gb=$ROOT_GB \
$ properties/cpu_arch='x86_64'

$ nova flavor-create baremetal auto $RAM $DISK_GB $CPU
$ nova flavor-key baremetal set cpu_arch=x86_64

Old parameter name	New parameter name
os_admin_username	admin_user
os_admin_password	admin_password
os_admin_tenant_name	admin_tenant_name

Feature	Plugin
Feature	Vanilla	HDP	Cloudera	Spark
Nova and Neutron network	x	x	x	x
Cluster Scaling	x	x	x	x
Swift Integration	x	x	x	x
Cinder Support	x	x	x	x
Data Locality	x	x	x	x
DNS	x	x	x	x
Kerberos		x	x
HDFS HA		x	x
EDP	x	x	x	x

OS	username
Ubuntu 12,14	ubuntu
Fedora	fedora
CentOS 6.x	cloud-user
CentOS 7.x	centos

OS	username
Ubuntu 14	ubuntu
Fedora 20	fedora
CentOS 6	cloud-user
CentOS 7	centos

Job type	Main binary	Libraries
`Hive`	required	optional
`Pig`	required	optional
`MapReduce`	not used	required
`MapReduce.Streaming`	not used	optional
`Java`	not used	required
`Shell`	required	optional
`Spark`	required	optional
`Storm`	required	not used
`Storm Pyelus`	required	not used

Name	Mapping Type	Location	Value Type	Required	Default
Example Class	args	0	string	false	teragen
Rows	args	1	number	true	unset
Output Path	args	2	data_source	false	hdfs://ip:port/path
Mapper Count	configs	mapred. map.tasks	number	false	unset

Name	Example Value
mapred.mapper.new-api	true
mapred.reducer.new-api	true
mapreduce.job.map.class	org.apache.oozie.example.SampleMapper
mapreduce.job.reduce.class	org.apache.oozie.example.SampleReducer

Name	Example Value
edp.streaming.mapper	/bin/cat
edp.streaming.reducer	/usr/bin/wc

Property	Type	Description
name	string	Config name.
description	string	A hint for user, what this config is used for.
config_type	enum	possible values are: ‘string’, ‘integer’, ‘boolean’, ‘enum’.
config_values	list	List of possible values, if config_type is enum.
default_value	string	Default value for config.
applicable_target	string	The target could be either a service returned by get_node_processes(...) call in form of ‘service:<service name>’, or ‘general’.
scope	enum	Could be either ‘node’ or ‘cluster’.
is_optional	bool	If is_optional is False and no default_value is specified, user must provide a value.
priority	int	1 or 2. A Hint for UI. Configs with priority 1 are always displayed. Priority 2 means user should click a button to see the config.

Property	Type	Description
config	config	A config object for which this user_input is provided.
value	...	Value for the config. Type depends on Config type.

Property	Type	Description
instance_id	string	Unique instance identifier.
instance_name	string	OpenStack instance name.
internal_ip	string	IP to communicate with other instances.
management_ip	string	IP of instance, accessible outside of internal network.
volumes	list	List of volumes attached to instance. Empty if ephemeral drive is used.
nova_info	object	Nova instance object.
username	string	Username, that sahara uses for establishing remote connections to instance.
hostname	string	Same as instance_name.
fqdn	string	Fully qualified domain name for this instance.
remote	helpers	Object with helpers for performing remote operations.

Property	Type	Description
name	string	Name of this Node Group in Cluster.
flavor_id	string	OpenStack Flavor used to boot instances.
image_id	string	Image id used to boot instances.
node_processes	list	List of processes running on each instance.
node_configs	dict	Configs dictionary, applied to instances.
volumes_per_node	int	Number of volumes mounted to each instance. 0 means use ephemeral drive.
volumes_size	int	Size of each volume (GB).
volumes_mount_prefix	string	Prefix added to mount path of each volume.
floating_ip_pool	string	Floating IP Pool name. All instances in the Node Group will have Floating IPs assigned from this pool.
count	int	Number of instances in this Node Group.
username	string	Username used by sahara to establish remote connections to instances.
configuration	dict	Merged dictionary of node configurations and cluster configurations.
storage_paths	list	List of directories where storage should be placed.

Property	Type	Description
name	string	Cluster name.
project_id	string	OpenStack Project id where this Cluster is available.
plugin_name	string	Plugin name.
hadoop_version	string	Hadoop version running on instances.
default_image_id	string	OpenStack image used to boot instances.
node_groups	list	List of Node Groups.
cluster_configs	dict	Dictionary of Cluster scoped configurations.
cluster_template_id	string	Cluster Template used for Node Groups and Configurations.
user_keypair_id	string	OpenStack keypair added to instances to make them accessible for user.
neutron_management_network	string	Neutron network ID. Instances will get fixed IPs in this network if ‘use_neutron’ config is set to True.
anti_affinity	list	List of processes that will be run on different hosts.
description	string	Cluster Description.
info	dict	Dictionary for additional information.

Property	Type	Description
config	config	A config object that is not valid.
error_message	string	Message that describes what exactly is wrong.

Job type	Configuration Values	Parameters	Arguments
`Hive`	Yes	Yes	No
`Pig`	Yes	Yes	Yes
`MapReduce`	Yes	No	No
`MapReduce.Streaming`	Yes	No	No
`Java`	Yes	No	Yes
`Shell`	Yes	Yes	Yes
`Spark`	Yes	No	Yes
`Storm`	Yes	No	Yes
`Storm Pyelus`	Yes	No	Yes

Name
fs.swift.service.sahara.username
fs.swift.service.sahara.password

Welcome to Sahara!¶

Overview¶

Rationale¶

Introduction¶

Details¶

General Workflow¶

User’s Perspective¶

Integration with Object Storage¶

Pluggable Deployment and Monitoring¶

Architecture¶

User guide¶

Sahara Installation Guide¶

To install with Fuel¶

To install with RDO¶

To install into a virtual environment¶

Common installation steps¶

Optional installation of default templates¶

Notes:¶

Sahara Configuration Guide¶

Basic configuration¶

Networking configuration¶

Floating IP management¶

Notifications configuration¶

Orchestration configuration¶

Policy configuration¶

Examples¶

API configuration¶

OpenStack Dashboard Configuration Guide¶

1. Networking¶

2. Different endpoint¶

Sahara Advanced Configuration Guide¶

Custom network topologies¶

DNS Hostname Resolution¶

Data-locality configuration¶

Distributed mode configuration¶

Distributed periodic tasks configuration¶

External key manager usage¶

Indirect instance access through proxy nodes¶

Multi region deployment¶

Non-root users¶

Object Storage access using proxy users¶

Detailed instructions¶

Volume instance locality configuration¶

Autoconfiguration for templates¶

NTP service configuration¶

CORS (Cross Origin Resource Sharing) Configuration¶

Cleanup time for incomplete clusters¶

Security Group Rules Configuration¶

Sahara Upgrade Guide¶

Icehouse -> Juno¶

Main binary renamed to sahara-all¶

Authentication middleware changes¶

Database package changes¶

Sahara integration into OpenStack Dashboard¶

Virtual machine user name changes¶

Anti affinity implementation changed¶

Juno -> Kilo¶

Sahara requires policy configuration¶

Kilo -> Liberty¶

Direct engine deprecation¶

Policy namespace changed (policy.json)¶

Liberty -> Mitaka¶

Mitaka -> Newton¶

Sample sahara.conf file¶

Getting Started¶

Clusters¶

Templates¶

Provisioning Plugins¶

Image Registry¶

Features¶

Sahara (Data Processing) UI User Guide¶

Launching a cluster via the sahara UI¶

Registering an Image¶

Create Node Group Templates¶

Create a Cluster Template¶

Launching a Cluster¶

Scaling a Cluster¶

Elastic Data Processing (EDP)¶

Data Sources¶

Job Binaries¶