Sitelet https://github.com/ii0/aws-cli/commit/f78cac98db74ced283f7d9dab0c69f4197c017e0
Skip to content

Commit f78cac9

Browse files
jameslsjamesls
committed
Add script to bundle an install
Only requires python
1 parent c5d7644 commit f78cac9

2 files changed

Lines changed: 221 additions & 0 deletions

File tree

‎scripts/install‎

Lines changed: 82 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,82 @@
1+
#!/usr/bin/env python
2+
import os
3+
import sys
4+
from contextlib import contextmanager
5+
import tempfile
6+
import subprocess
7+
8+
PACKAGES_DIR = os.path.join(
9+
os.path.dirname(os.path.abspath(__file__)), 'packages')
10+
INSTALL_DIR = os.path.expanduser('~/.local/lib/aws')
11+
12+
13+
class BadRCError(Exception):
14+
pass
15+
16+
17+
@contextmanager
18+
def cd(dirname):
19+
original = os.getcwd()
20+
os.chdir(dirname)
21+
try:
22+
yield
23+
finally:
24+
os.chdir(original)
25+
26+
27+
def run(cmd):
28+
sys.stdout.write("Running cmd: %s\n" % cmd)
29+
p = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE,
30+
stderr=subprocess.PIPE)
31+
rc = p.wait()
32+
if rc != 0:
33+
raise BadRCError("Bad rc (%s) for cmd '%s': %s" % (
34+
rc, cmd, p.stderr.read() + p.stdout.read()))
35+
return p.stdout.read()
36+
37+
38+
def create_install_structure(working_dir):
39+
if not os.path.isdir(INSTALL_DIR):
40+
os.makedirs(INSTALL_DIR)
41+
_create_virtualenv(location=INSTALL_DIR, working_dir=working_dir)
42+
43+
44+
def _create_virtualenv(location, working_dir):
45+
# working_dir is used (generally somewhere in /tmp) so that we
46+
# don't modify the install/packages directories.
47+
with cd(PACKAGES_DIR):
48+
venv = [p for p in os.listdir('.') if p.startswith('virtualenv')][0]
49+
run('tar xvfz %s -C %s' % (venv, working_dir))
50+
with cd(working_dir):
51+
# We know that virtualenv is the only dir in this directory
52+
# so we can listdir()[0] it.
53+
with cd(os.listdir('.')[0]):
54+
run('%s virtualenv.py --python %s %s' % (sys.executable,
55+
sys.executable,
56+
location))
57+
58+
59+
def create_working_dir():
60+
d = tempfile.mkdtemp()
61+
return d
62+
63+
64+
def pip_install_packages():
65+
cli_tarball = [p for p in os.listdir(PACKAGES_DIR) if p.startswith('awscli')]
66+
assert len(cli_tarball) == 1
67+
cli_tarball = cli_tarball[0]
68+
pip_script = os.path.join(INSTALL_DIR, 'bin', 'pip')
69+
with cd(PACKAGES_DIR):
70+
run('%s install --no-index --find-links file://%s %s' % (
71+
pip_script, PACKAGES_DIR, cli_tarball))
72+
73+
74+
def main():
75+
working_dir = create_working_dir()
76+
create_install_structure(working_dir)
77+
pip_install_packages()
78+
print("You can now run: %s --version" % os.path.join(INSTALL_DIR, 'bin', 'aws'))
79+
80+
81+
if __name__ == '__main__':
82+
main()

‎scripts/make-bundle‎

Lines changed: 139 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,139 @@
1+
#!/usr/bin/env python
2+
"""Script to create self contained install.
3+
4+
The goal of this script is simple:
5+
6+
* Create a self contained install of the CLI that
7+
has requires no external resources during installation.
8+
9+
It does this by using all the normal python tooling
10+
(virtualenv, pip) but provides a simple, easy to use
11+
interface for those not familiar with the python
12+
ecosystem.
13+
14+
"""
15+
import os
16+
import sys
17+
import subprocess
18+
import shutil
19+
import tempfile
20+
from contextlib import contextmanager
21+
# Assuming you have the awscli deps installed, so
22+
# using requests is ok.
23+
import requests
24+
# These are package versions that are needed to boostrap
25+
# the installation process. It is *not* the deps required
26+
# by awscli/botocore.
27+
PACKAGE_VERSION = {
28+
'virtualenv': '1.10.1',
29+
}
30+
31+
32+
class BadRCError(Exception):
33+
pass
34+
35+
36+
@contextmanager
37+
def cd(dirname):
38+
original = os.getcwd()
39+
os.chdir(dirname)
40+
try:
41+
yield
42+
finally:
43+
os.chdir(original)
44+
45+
46+
def run(cmd):
47+
sys.stdout.write("Running cmd: %s\n" % cmd)
48+
p = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE,
49+
stderr=subprocess.PIPE)
50+
rc = p.wait()
51+
if rc != 0:
52+
raise BadRCError("Bad rc (%s) for cmd '%s': %s" % (
53+
rc, cmd, p.stderr.read() + p.stdout.read()))
54+
return p.stdout.read()
55+
56+
57+
def create_scratch_dir():
58+
# This creates the dir where all the bundling occurs.
59+
# First we need a top level dir.
60+
dirname = tempfile.mkdtemp(prefix='bundle')
61+
# Then we need to create a dir where all the packages
62+
# will come from.
63+
os.mkdir(os.path.join(dirname, 'packages'))
64+
return dirname
65+
66+
67+
def download_package_tarballs(dirname, packages):
68+
with cd(dirname):
69+
for package in packages:
70+
run('pip install -d . %s==%s' % (
71+
package, PACKAGE_VERSION[package]))
72+
73+
74+
def download_cli_deps(scratch_dir):
75+
# As far as i can tell this is a bug in pip.
76+
# Running 'pip intall -d . ~/clidir' does not work.
77+
# Running 'pip install -d foo ~/clidir' results
78+
# in python consuming 100% CPU.
79+
# This problem is surprisingly hard, specifically that
80+
# of downloading all the dependent python packages of
81+
# the awscli as tarballs.
82+
# So here's what we do.
83+
# Step one create a virtualenv.
84+
venv_dir = '/tmp/venv-%s' % os.path.basename(scratch_dir)
85+
run('virtualenv %s' % venv_dir)
86+
pip = os.path.join(venv_dir, 'bin', 'pip')
87+
assert os.path.isfile(pip)
88+
awscli_dir = os.path.dirname(
89+
os.path.dirname(os.path.abspath(__file__)))
90+
with cd(scratch_dir):
91+
run('%s install -d . -e %s' % (pip, awscli_dir))
92+
# Remove the awscli package, we'll create our own sdist.
93+
_remove_cli_zip(scratch_dir)
94+
95+
96+
def _remove_cli_zip(scratch_dir):
97+
clidir = [f for f in os.listdir(scratch_dir) if f.startswith('awscli')]
98+
assert len(clidir) == 1
99+
os.remove(os.path.join(scratch_dir, clidir[0]))
100+
101+
102+
def add_cli_sdist(scratch_dir):
103+
awscli_dir = os.path.dirname(
104+
os.path.dirname(os.path.abspath(__file__)))
105+
if os.path.exists(os.path.join(awscli_dir, 'dist')):
106+
shutil.rmtree(os.path.join(awscli_dir, 'dist'))
107+
with cd(awscli_dir):
108+
run('python setup.py sdist')
109+
filename = os.listdir('dist')[0]
110+
os.rename(os.path.join('dist', filename),
111+
os.path.join(scratch_dir, filename))
112+
113+
114+
def create_bootstrap_script(scratch_dir):
115+
install_script = os.path.join(
116+
os.path.dirname(os.path.abspath(__file__)), 'install')
117+
shutil.copy(install_script, os.path.join(scratch_dir, 'install'))
118+
119+
120+
def zip_dir(scratch_dir):
121+
pass
122+
123+
124+
def main():
125+
scratch_dir = create_scratch_dir()
126+
package_dir = os.path.join(scratch_dir, 'packages')
127+
print("Bundle dir at: %s" % scratch_dir)
128+
download_package_tarballs(
129+
package_dir, packages=['virtualenv'])
130+
download_cli_deps(package_dir)
131+
add_cli_sdist(package_dir)
132+
create_bootstrap_script(scratch_dir)
133+
zip_dir(scratch_dir)
134+
print("Bundled install is at: %s" % scratch_dir)
135+
print("You can cd to this directory and run './install'")
136+
137+
138+
if __name__ == '__main__':
139+
main()

0 commit comments

Comments
 (0)