#!/usr/bin/env python

"""Script to create self contained install.

The goal of this script is simple:

* Create a self contained install of the CLI that

has requires no external resources during installation.

It does this by using all the normal python tooling

(virtualenv, pip) but provides a simple, easy to use

interface for those not familiar with the python

ecosystem.

"""

import os

import sys

import time

import subprocess

import shutil

import tempfile

import zipfile

from contextlib import contextmanager

# These are package versions that are needed to boostrap

# the installation process. It is *not* the deps required

# by awscli/botocore.

PACKAGE_VERSION = {

'virtualenv': '15.1.0',

# These packages are included because they are required for

# python2.6, but our normal dependency downloading via pip

# only works if you use python2.6. To fix this issue, we're

# explicitly saying that we need to download these packages

# even if pip doesn't think we need them. That way we can

# run make-bundle in python versions besides 2.6.

'ordereddict': '1.1',

'simplejson': '3.3.0',

'argparse': '1.2.1',

'python-dateutil': '2.6.1',

'setuptools-scm': '1.15.7',

}

INSTALL_ARGS = '--allow-all-external --no-use-wheel'

class BadRCError(Exception):

pass

@contextmanager

def cd(dirname):

original = os.getcwd()

os.chdir(dirname)

try:

yield

finally:

os.chdir(original)

def run(cmd):

sys.stdout.write("Running cmd: %s\n" % cmd)

p = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE,

stderr=subprocess.PIPE)

stdout, stderr = p.communicate()

rc = p.wait()

if p.returncode != 0:

raise BadRCError("Bad rc (%s) for cmd '%s': %s" % (

rc, cmd, stderr + stdout))

return stdout

def create_scratch_dir():

# This creates the dir where all the bundling occurs.

# First we need a top level dir.

dirname = tempfile.mkdtemp(prefix='bundle')

# Then we need to create a dir where all the packages

# will come from.

os.mkdir(os.path.join(dirname, 'packages'))

os.mkdir(os.path.join(dirname, 'packages', 'setup'))

return dirname

def download_package_tarballs(dirname, packages):

with cd(dirname):

for package in packages:

run('pip install -d . %s==%s %s' % (

package, PACKAGE_VERSION[package], INSTALL_ARGS))

def download_cli_deps(scratch_dir):

# As far as i can tell this is a bug in pip.

# Running 'pip intall -d . ~/clidir' does not work.

# Running 'pip install -d foo ~/clidir' results

# in python consuming 100% CPU.

# This problem is surprisingly hard, specifically that

# of downloading all the dependent python packages of

# the awscli as tarballs.

# So here's what we do.

# Step one create a virtualenv.

venv_dir = tempfile.mkdtemp()

try:

run('virtualenv --no-download %s' % venv_dir)

pip = os.path.join(venv_dir, 'bin', 'pip')

assert os.path.isfile(pip)

awscli_dir = os.path.dirname(

os.path.dirname(os.path.abspath(__file__)))

with cd(scratch_dir):

run('%s install %s -d . -e %s' % (pip, INSTALL_ARGS, awscli_dir))

# Remove the awscli package, we'll create our own sdist.

_remove_cli_zip(scratch_dir)

finally:

shutil.rmtree(venv_dir)

def _remove_cli_zip(scratch_dir):

clidir = [f for f in os.listdir(scratch_dir) if f.startswith('awscli')]

assert len(clidir) == 1

os.remove(os.path.join(scratch_dir, clidir[0]))

def add_cli_sdist(scratch_dir):

awscli_dir = os.path.dirname(

os.path.dirname(os.path.abspath(__file__)))

if os.path.exists(os.path.join(awscli_dir, 'dist')):

shutil.rmtree(os.path.join(awscli_dir, 'dist'))

with cd(awscli_dir):

run('python setup.py sdist')

filename = os.listdir('dist')[0]

shutil.move(os.path.join('dist', filename),

os.path.join(scratch_dir, filename))

def create_bootstrap_script(scratch_dir):

install_script = os.path.join(

os.path.dirname(os.path.abspath(__file__)), 'install')

shutil.copy(install_script, os.path.join(scratch_dir, 'install'))

def zip_dir(scratch_dir):

basename = 'awscli-bundle.zip'

dirname, tmpdir = os.path.split(scratch_dir)

final_dir_name = os.path.join(dirname, 'awscli-bundle')

if os.path.isdir(final_dir_name):

shutil.rmtree(final_dir_name)

shutil.move(scratch_dir, final_dir_name)

with cd(dirname):

with zipfile.ZipFile(basename, 'w', zipfile.ZIP_DEFLATED) as zipped:

for root, dirnames, filenames in os.walk('awscli-bundle'):

for filename in filenames:

zipped.write(os.path.join(root, filename))

return os.path.join(dirname, basename)

def verify_preconditions():

# The pip version looks like:

# 'pip 1.4.1 from ....'

pip_version = run('pip --version').strip().split()[1]

# Virtualenv version just has the version string: '1.14.5\n'

virtualenv_version = run('virtualenv --version').strip()

_min_version_required('9.0.1', pip_version, 'pip')

_min_version_required('15.1.0', virtualenv_version, 'virtualenv')

def _min_version_required(min_version, actual_version, name):

# precondition: min_version is major.minor.patch

# actual_version is major.minor.patch

min_split = min_version.split('.')

actual_split = actual_version.decode('utf-8').split('.')

for min_version_part, actual_version_part in zip(min_split, actual_split):

if int(actual_version_part) >= int(min_version_part):

return

raise ValueError("%s requires at least version %s, but version %s was "

"found." % (name, min_version, actual_version))

def main():

verify_preconditions()

scratch_dir = create_scratch_dir()

package_dir = os.path.join(scratch_dir, 'packages')

print("Bundle dir at: %s" % scratch_dir)

download_package_tarballs(

package_dir,

packages=['virtualenv', 'ordereddict', 'simplejson', 'argparse', 'python-dateutil'])

# Some packages require setup time dependencies, and so we will need to

# manually install them. We isolate them to a particular directory so we

# can run the install before the things they're dependent on. We have to do

# this because pip won't actually find them since it doesn't handle build

# dependencies.

setup_dir = os.path.join(package_dir, 'setup')

download_package_tarballs(

setup_dir,

packages=['setuptools-scm']

)

download_cli_deps(package_dir)

add_cli_sdist(package_dir)

create_bootstrap_script(scratch_dir)

zip_filename = zip_dir(scratch_dir)

print("Zipped bundle installer is at: %s" % zip_filename)

if __name__ == '__main__':

main()