forked from Kaggle/docker-python
-
Notifications
You must be signed in to change notification settings - Fork 0
Expand file tree
/
Copy pathtest
More file actions
executable file
·118 lines (107 loc) · 3.98 KB
/
Copy pathtest
File metadata and controls
executable file
·118 lines (107 loc) · 3.98 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
#!/bin/bash
set -e
IMAGE_TAG='kaggle/python-build'
IMAGE_TAG_OVERRIDE=''
ADDITONAL_OPTS='--runtime runc ' # Use the CPU runtime by default
PATTERN='test*.py'
usage() {
cat << EOF
Usage: $0 [OPTIONS]
Run tests for a newly-built Python Docker image.
By default, it runs the tests for the CPU image.
Options:
-g, --gpu Run tests for the GPU image.
-i, --image IMAGE Run tests against the specified image
-p, --pattern PATTERN Pattern to match test files ($PATTERN default)
EOF
}
while :; do
case "$1" in
-h|--help)
usage
exit
;;
-g|--gpu)
IMAGE_TAG='kaggle/python-gpu-build'
ADDITONAL_OPTS='--runtime nvidia -v /tmp/empty_dir:/usr/local/cuda/lib64/stubs:ro'
;;
-i|--image)
if [[ -z $2 ]]; then
usage
printf 'ERROR: No IMAGE specified after the %s flag.\n' "$1" >&2
exit
fi
IMAGE_TAG_OVERRIDE=$2
shift # skip the flag value
;;
-p|--pattern)
if [[ -z $2 ]]; then
usage
printf 'ERROR: No PATTERN specified after the %s flag.\n' "$1" >&2
exit
fi
PATTERN="$2"
shift # skip the flag value
;;
-?*)
usage
printf 'ERROR: Unknown option: %s\n' "$1" >&2
exit
;;
*)
break
esac
shift
done
if [[ -n "$IMAGE_TAG_OVERRIDE" ]]; then
IMAGE_TAG="$IMAGE_TAG_OVERRIDE"
fi
readonly IMAGE_TAG
readonly ADDITONAL_OPTS
readonly PATTERN
set -x
mkdir -p /tmp/python-build/tmp
mkdir -p /tmp/python-build/devshm
mkdir -p /tmp/python-build/working
mkdir -p /tmp/python-build/kaggle
# Only run Jupyter server test if no specific test pattern is specified.
if [ $PATTERN == 'test*.py' ]; then
# Check that Jupyter server can run; if it dies on startup, the `docker kill` command will throw an error
docker run -d --name=jupyter_test --read-only --net=none -e HOME=/tmp -v $PWD:/input:ro -v /tmp/python-build/working:/working -w=/working -v /tmp/python-build/tmp:/tmp -v /tmp/python-build/devshm:/dev/shm "$IMAGE_TAG" jupyter notebook --allow-root --ip="*"
sleep 3
docker kill jupyter_test && docker rm jupyter_test
fi
# Note about `TF_FORCE_GPU_ALLOW_GROWTH`. This allocate memory at runtime as needed.
# By default, TensorFlow maps nearly all of the GPU memory visible to the process.
# This is causing issue when other libraries are trying to run tests using a GPU.
# See: https://www.tensorflow.org/guide/gpu#allowing_gpu_memory_growth
#
# Note about `XLA_PYTHON_CLIENT_PREALLOCATE`. By default, JAX preallocates 90%
# of the GPU memory which is causing issues when other libraries are trying to run
# tests using a GPU.
# See: https://jax.readthedocs.io/en/latest/gpu_memory_allocation.html
#
# Note about `--hostname localhost` (b/158137436)
# hostname defaults to the container name which fails DNS name
# resolution with --net=none (required to keep tests hermetic). See details in bug.
#
# Note about CLOUDSDK_CONFIG=/tmp/.config/gcloud
# We use the /tmp dir since the filesystem is --read-only and we need writable space for gcloud configs.
docker run --rm -t --read-only --net=none \
-e HOME=/tmp -e KAGGLE_DATA_PROXY_TOKEN=test-key \
-e KAGGLE_USER_SECRETS_TOKEN_KEY=test-secrets-key \
-e KAGGLE_URL_BASE=http://127.0.0.1:0 \
-e KAGGLE_DATA_PROXY_URL=http://127.0.0.1:8000 \
-e KAGGLE_DATA_PROXY_PROJECT=test \
-e TF_FORCE_GPU_ALLOW_GROWTH=true \
-e XLA_PYTHON_CLIENT_PREALLOCATE=false \
-e CLOUDSDK_CONFIG=/tmp/.config/gcloud \
--hostname localhost \
--shm-size=2g \
-v $PWD:/input:ro -v /tmp/python-build/working:/working \
-v /tmp/python-build/tmp:/tmp -v /tmp/python-build/devshm:/dev/shm \
-v /tmp/python-build/kaggle:/kaggle \
-w=/working \
$ADDITONAL_OPTS \
"$IMAGE_TAG" \
/bin/bash -c "python -m unittest discover -s /input/tests -p $PATTERN -v"