CI integration

13 files changed, 448 insertions, 0 deletions

diff --git a/.ci/appveyor.yml b/.ci/appveyor.yml
new file mode 100644
index 0000000..318b16b
--- /dev/null
+++ b/.ci/appveyor.yml
@@ -0,0 +1,51 @@
+environment:
+    global:
+        S3_UPLOAD_USERNAME: oss-ci-bot
+        S3_UPLOAD_BUCKET: magicstack-oss-releases
+        S3_UPLOAD_ACCESSKEY:
+            secure: 1vmOqSXq5zDN8UdezZ3H4l0A9LUJiTr7Wuy9whCdffE=
+        S3_UPLOAD_SECRET:
+            secure: XudOvV6WtY9yRoqKahXMswFth8SF1UTnSXws4UBjeqzQUjOx2V2VRvIdpPfiqUKt
+
+    matrix:
+        - PYTHON: "C:\\Python35\\python.exe"
+        - PYTHON: "C:\\Python35-x64\\python.exe"
+        - PYTHON: "C:\\Python36\\python.exe"
+        - PYTHON: "C:\\Python36-x64\\python.exe"
+
+branches:
+    # Avoid building PR branches.
+    only:
+        - master
+        - ci
+        - releases
+
+install:
+    - "%PYTHON% -m pip install --upgrade pip wheel setuptools"
+    - "%PYTHON% -m pip install --upgrade -r .ci/requirements-win.txt"
+
+build_script:
+    - "%PYTHON% setup.py build_ext --inplace"
+
+test_script:
+    - "%PYTHON% setup.py test"
+
+after_test:
+    - "%PYTHON% setup.py bdist_wheel"
+
+artifacts:
+    - path: dist\*
+
+deploy_script:
+    - ps: |
+        if ($env:appveyor_repo_branch -eq 'releases') {
+            $PACKAGE_VERSION = & "$env:PYTHON" ".ci/package-version.py"
+            $PYPI_VERSION = & "$env:PYTHON" ".ci/pypi-check.py" "immutables"
+
+            if ($PACKAGE_VERSION -eq $PYPI_VERSION) {
+                Write-Error "immutables-$PACKAGE_VERSION is already published on PyPI"
+                exit 1
+            }
+
+            & "$env:PYTHON" ".ci/s3-upload.py" dist\*.whl
+        }
diff --git a/.ci/build-manylinux-wheels.sh b/.ci/build-manylinux-wheels.sh
new file mode 100755
index 0000000..6d78104
--- /dev/null
+++ b/.ci/build-manylinux-wheels.sh
@@ -0,0 +1,29 @@
+#!/bin/bash
+
+set -e -x
+
+# Compile wheels
+PYTHON="/opt/python/${PYTHON_VERSION}/bin/python"
+PIP="/opt/python/${PYTHON_VERSION}/bin/pip"
+${PIP} install --upgrade pip wheel
+${PIP} install --upgrade setuptools
+${PIP} install -r /io/.ci/requirements.txt
+make -C /io/ PYTHON="${PYTHON}"
+${PIP} wheel /io/ -w /io/dist/
+
+# Bundle external shared libraries into the wheels.
+for whl in /io/dist/*.whl; do
+    auditwheel repair $whl -w /io/dist/
+    rm /io/dist/*-linux_*.whl
+done
+
+# Grab docker host, where Postgres should be running.
+export PGHOST=$(ip route | awk '/default/ { print $3 }' | uniq)
+export PGUSER="postgres"
+
+PYTHON="/opt/python/${PYTHON_VERSION}/bin/python"
+PIP="/opt/python/${PYTHON_VERSION}/bin/pip"
+${PIP} install ${PYMODULE} --no-index -f file:///io/dist
+rm -rf /io/tests/__pycache__
+make -C /io/ PYTHON="${PYTHON}" testinstalled
+rm -rf /io/tests/__pycache__
diff --git a/.ci/package-version.py b/.ci/package-version.py
new file mode 100755
index 0000000..bbf2ff4
--- /dev/null
+++ b/.ci/package-version.py
@@ -0,0 +1,26 @@
+#!/usr/bin/env python3
+
+
+import os.path
+import sys
+
+
+def main():
+    version_file = os.path.join(
+        os.path.dirname(os.path.dirname(__file__)),
+        'immutables', '__init__.py')
+
+    with open(version_file, 'r') as f:
+        for line in f:
+            if line.startswith('__version__ ='):
+                _, _, version = line.partition('=')
+                print(version.strip(" \n'\""))
+                return 0
+
+    print('could not find package version in immutables/__init__.py',
+          file=sys.stderr)
+    return 1
+
+
+if __name__ == '__main__':
+    sys.exit(main())
diff --git a/.ci/pypi-check.py b/.ci/pypi-check.py
new file mode 100755
index 0000000..1b9c11c
--- /dev/null
+++ b/.ci/pypi-check.py
@@ -0,0 +1,30 @@
+#!/usr/bin/env python3
+
+
+import argparse
+import sys
+import xmlrpc.client
+
+
+def main():
+    parser = argparse.ArgumentParser(description='PyPI package checker')
+    parser.add_argument('package_name', metavar='PACKAGE-NAME')
+
+    parser.add_argument(
+        '--pypi-index-url',
+        help=('PyPI index URL.'),
+        default='https://pypi.python.org/pypi')
+
+    args = parser.parse_args()
+
+    pypi = xmlrpc.client.ServerProxy(args.pypi_index_url)
+    releases = pypi.package_releases(args.package_name)
+
+    if releases:
+        print(next(iter(sorted(releases, reverse=True))))
+
+    return 0
+
+
+if __name__ == '__main__':
+    sys.exit(main())
diff --git a/.ci/requirements-win.txt b/.ci/requirements-win.txt
new file mode 100644
index 0000000..791da6b
--- /dev/null
+++ b/.ci/requirements-win.txt
@@ -0,0 +1 @@
+tinys3
diff --git a/.ci/requirements.txt b/.ci/requirements.txt
new file mode 100644
index 0000000..403ef59
--- /dev/null
+++ b/.ci/requirements.txt
@@ -0,0 +1,2 @@
+tinys3
+twine
diff --git a/.ci/s3-download-release.py b/.ci/s3-download-release.py
new file mode 100755
index 0000000..223f7f1
--- /dev/null
+++ b/.ci/s3-download-release.py
@@ -0,0 +1,74 @@
+#!/usr/bin/env python3
+
+
+import argparse
+import os
+import os.path
+import sys
+import urllib.request
+
+import tinys3
+
+
+def main():
+    parser = argparse.ArgumentParser(description='S3 File Uploader')
+    parser.add_argument(
+        '--s3-bucket',
+        help=('S3 bucket name (defaults to $S3_UPLOAD_BUCKET)'),
+        default=os.environ.get('S3_UPLOAD_BUCKET'))
+    parser.add_argument(
+        '--s3-region',
+        help=('S3 region (defaults to $S3_UPLOAD_REGION)'),
+        default=os.environ.get('S3_UPLOAD_REGION'))
+    parser.add_argument(
+        '--s3-username',
+        help=('S3 username (defaults to $S3_UPLOAD_USERNAME)'),
+        default=os.environ.get('S3_UPLOAD_USERNAME'))
+    parser.add_argument(
+        '--s3-key',
+        help=('S3 access key (defaults to $S3_UPLOAD_ACCESSKEY)'),
+        default=os.environ.get('S3_UPLOAD_ACCESSKEY'))
+    parser.add_argument(
+        '--s3-secret',
+        help=('S3 secret (defaults to $S3_UPLOAD_SECRET)'),
+        default=os.environ.get('S3_UPLOAD_SECRET'))
+    parser.add_argument(
+        '--destdir',
+        help='Destination directory.')
+    parser.add_argument(
+        'package', metavar='PACKAGE',
+        help='Package name and version to download.')
+
+    args = parser.parse_args()
+
+    if args.s3_region:
+        endpoint = 's3-{}.amazonaws.com'.format(args.s3_region.lower())
+    else:
+        endpoint = 's3.amazonaws.com'
+
+    conn = tinys3.Connection(
+        access_key=args.s3_key,
+        secret_key=args.s3_secret,
+        default_bucket=args.s3_bucket,
+        tls=True,
+        endpoint=endpoint,
+    )
+
+    files = []
+
+    for entry in conn.list(args.package):
+        files.append(entry['key'])
+
+    destdir = args.destdir or os.getpwd()
+
+    for file in files:
+        print('Downloading {}...'.format(file))
+        url = 'https://{}/{}/{}'.format(endpoint, args.s3_bucket, file)
+        target = os.path.join(destdir, file)
+        urllib.request.urlretrieve(url, target)
+
+    return 0
+
+
+if __name__ == '__main__':
+    sys.exit(main())
diff --git a/.ci/s3-upload.py b/.ci/s3-upload.py
new file mode 100755
index 0000000..92479af
--- /dev/null
+++ b/.ci/s3-upload.py
@@ -0,0 +1,62 @@
+#!/usr/bin/env python3
+
+
+import argparse
+import glob
+import os
+import os.path
+import sys
+
+import tinys3
+
+
+def main():
+    parser = argparse.ArgumentParser(description='S3 File Uploader')
+    parser.add_argument(
+        '--s3-bucket',
+        help=('S3 bucket name (defaults to $S3_UPLOAD_BUCKET)'),
+        default=os.environ.get('S3_UPLOAD_BUCKET'))
+    parser.add_argument(
+        '--s3-region',
+        help=('S3 region (defaults to $S3_UPLOAD_REGION)'),
+        default=os.environ.get('S3_UPLOAD_REGION'))
+    parser.add_argument(
+        '--s3-username',
+        help=('S3 username (defaults to $S3_UPLOAD_USERNAME)'),
+        default=os.environ.get('S3_UPLOAD_USERNAME'))
+    parser.add_argument(
+        '--s3-key',
+        help=('S3 access key (defaults to $S3_UPLOAD_ACCESSKEY)'),
+        default=os.environ.get('S3_UPLOAD_ACCESSKEY'))
+    parser.add_argument(
+        '--s3-secret',
+        help=('S3 secret (defaults to $S3_UPLOAD_SECRET)'),
+        default=os.environ.get('S3_UPLOAD_SECRET'))
+    parser.add_argument(
+        'files', nargs='+', metavar='FILE', help='Files to upload')
+
+    args = parser.parse_args()
+
+    if args.s3_region:
+        endpoint = 's3-{}.amazonaws.com'.format(args.s3_region.lower())
+    else:
+        endpoint = 's3.amazonaws.com'
+
+    conn = tinys3.Connection(
+        access_key=args.s3_key,
+        secret_key=args.s3_secret,
+        default_bucket=args.s3_bucket,
+        tls=True,
+        endpoint=endpoint,
+    )
+
+    for pattern in args.files:
+        for fn in glob.iglob(pattern):
+            with open(fn, 'rb') as f:
+                conn.upload(os.path.basename(fn), f)
+
+    return 0
+
+
+if __name__ == '__main__':
+    sys.exit(main())
diff --git a/.ci/travis-before-install.sh b/.ci/travis-before-install.sh
new file mode 100755
index 0000000..fdc9202
--- /dev/null
+++ b/.ci/travis-before-install.sh
@@ -0,0 +1,16 @@
+#!/bin/bash
+
+set -e -x
+
+if [ "${TRAVIS_OS_NAME}" == "osx" ]; then
+    git clone --depth 1 https://github.com/yyuu/pyenv.git ~/.pyenv
+    PYENV_ROOT="$HOME/.pyenv"
+    PATH="$PYENV_ROOT/bin:$PATH"
+    eval "$(pyenv init -)"
+
+    if ! (pyenv versions | grep "${PYTHON_VERSION}$"); then
+        pyenv install ${PYTHON_VERSION}
+    fi
+    pyenv global ${PYTHON_VERSION}
+    pyenv rehash
+fi
diff --git a/.ci/travis-build-wheels.sh b/.ci/travis-build-wheels.sh
new file mode 100755
index 0000000..92771bd
--- /dev/null
+++ b/.ci/travis-build-wheels.sh
@@ -0,0 +1,73 @@
+#!/bin/bash
+
+set -e -x
+
+
+if [[ "${TRAVIS_BRANCH}" != "releases" || "${BUILD}" != *wheels* ]]; then
+    # Not a release
+    exit 0
+fi
+
+
+if [ "${TRAVIS_OS_NAME}" == "osx" ]; then
+    PYENV_ROOT="$HOME/.pyenv"
+    PATH="$PYENV_ROOT/bin:$PATH"
+    eval "$(pyenv init -)"
+fi
+
+PACKAGE_VERSION=$(python ".ci/package-version.py")
+PYPI_VERSION=$(python ".ci/pypi-check.py" "${PYMODULE}")
+
+if [ "${PACKAGE_VERSION}" == "${PYPI_VERSION}" ]; then
+    echo "${PYMODULE}-${PACKAGE_VERSION} is already published on PyPI"
+    exit 1
+fi
+
+
+pushd $(dirname $0) > /dev/null
+_root=$(dirname $(pwd -P))
+popd > /dev/null
+
+
+_upload_wheels() {
+    python "${_root}/.ci/s3-upload.py" "${_root}/dist"/*.whl
+    sudo rm -rf "${_root}/dist"/*.whl
+}
+
+
+if [ "${TRAVIS_OS_NAME}" == "linux" ]; then
+    for pyver in ${RELEASE_PYTHON_VERSIONS}; do
+        ML_PYTHON_VERSION=$(python3 -c \
+            "print('cp{maj}{min}-cp{maj}{min}m'.format( \
+                   maj='${pyver}'.split('.')[0], \
+                   min='${pyver}'.split('.')[1]))")
+
+        for arch in x86_64 i686; do
+            ML_IMAGE="quay.io/pypa/manylinux1_${arch}"
+            docker pull "${ML_IMAGE}"
+            docker run --rm \
+                -v "${_root}":/io \
+                -e "PYMODULE=${PYMODULE}" \
+                -e "PYTHON_VERSION=${ML_PYTHON_VERSION}" \
+                "${ML_IMAGE}" /io/.ci/build-manylinux-wheels.sh
+
+            _upload_wheels
+        done
+    done
+
+elif [ "${TRAVIS_OS_NAME}" == "osx" ]; then
+    export PGINSTALLATION="/usr/local/opt/postgresql@${PGVERSION}/bin"
+
+    make clean && make -C "${_root}"
+    pip wheel "${_root}" -w "${_root}/dist/"
+
+    pip install ${PYMODULE} --no-index -f "file:///${_root}/dist"
+    pushd / >/dev/null
+    make -C "${_root}" testinstalled
+    popd >/dev/null
+
+    _upload_wheels
+
+else
+    echo "Cannot build on ${TRAVIS_OS_NAME}."
+fi
diff --git a/.ci/travis-install.sh b/.ci/travis-install.sh
new file mode 100755
index 0000000..e9715ee
--- /dev/null
+++ b/.ci/travis-install.sh
@@ -0,0 +1,13 @@
+#!/bin/bash
+
+set -e -x
+
+if [ "${TRAVIS_OS_NAME}" == "osx" ]; then
+    PYENV_ROOT="$HOME/.pyenv"
+    PATH="$PYENV_ROOT/bin:$PATH"
+    eval "$(pyenv init -)"
+fi
+
+pip install --upgrade pip wheel
+pip install --upgrade setuptools
+pip install --upgrade -r .ci/requirements.txt
diff --git a/.ci/travis-release.sh b/.ci/travis-release.sh
new file mode 100755
index 0000000..60d1c62
--- /dev/null
+++ b/.ci/travis-release.sh
@@ -0,0 +1,55 @@
+#!/bin/bash
+
+set -e -x
+
+
+if [ -z "${TRAVIS_TAG}" ]; then
+    # Not a release
+    exit 0
+fi
+
+
+PACKAGE_VERSION=$(python ".ci/package-version.py")
+PYPI_VERSION=$(python ".ci/pypi-check.py" "${PYMODULE}")
+
+if [ "${PACKAGE_VERSION}" == "${PYPI_VERSION}" ]; then
+    echo "${PYMODULE}-${PACKAGE_VERSION} is already published on PyPI"
+    exit 0
+fi
+
+# Check if all expected wheels have been built and uploaded.
+release_platforms=(
+    "macosx_10_??_x86_64"
+    "manylinux1_i686"
+    "manylinux1_x86_64"
+    "win32"
+    "win_amd64"
+)
+
+P="${PYMODULE}-${PACKAGE_VERSION}"
+expected_wheels=()
+
+for pyver in ${RELEASE_PYTHON_VERSIONS}; do
+    pyver="${pyver//./}"
+    abitag="cp${pyver}-cp${pyver}m"
+    for plat in "${release_platforms[@]}"; do
+        expected_wheels+=("${P}-${abitag}-${plat}.whl")
+    done
+done
+
+rm -rf dist/*.whl dist/*.tar.*
+python setup.py sdist
+python ".ci/s3-download-release.py" --destdir=dist/ "${P}"
+
+_file_exists() { [[ -f $1 ]]; }
+
+for distfile in "${expected_wheels[@]}"; do
+    if ! _file_exists dist/${distfile}; then
+        echo "Expected wheel ${distfile} not found."
+        exit 1
+    fi
+done
+
+python -m twine upload dist/*.whl dist/*.tar.*
+
+.ci/travis-publish-docs.sh
diff --git a/.ci/travis-tests.sh b/.ci/travis-tests.sh
new file mode 100755
index 0000000..bf8f5ad
--- /dev/null
+++ b/.ci/travis-tests.sh
@@ -0,0 +1,16 @@
+#!/bin/bash
+
+set -e -x
+
+if [[ "${BUILD}" != *tests* ]]; then
+    echo "Skipping tests."
+    exit 0
+fi
+
+if [ "${TRAVIS_OS_NAME}" == "osx" ]; then
+    PYENV_ROOT="$HOME/.pyenv"
+    PATH="$PYENV_ROOT/bin:$PATH"
+    eval "$(pyenv init -)"
+fi
+
+python setup.py test