diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 000000000..ff39b79b0 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,2 @@ +.git +mason_packages \ No newline at end of file diff --git a/scripts/clang++/4.0.1/script.sh b/scripts/clang++/4.0.1/script.sh index 391259e3c..c21f18d8a 100755 --- a/scripts/clang++/4.0.1/script.sh +++ b/scripts/clang++/4.0.1/script.sh @@ -1,5 +1,7 @@ #!/usr/bin/env bash +# For context on this file see https://github.com/mapbox/mason/blob/master/scripts/llvm/base/README.md + # dynamically determine the path to this package HERE="$( cd "$( dirname "${BASH_SOURCE[0]}" )" > /dev/null && pwd )" diff --git a/scripts/clang-format/4.0.1/script.sh b/scripts/clang-format/4.0.1/script.sh index 391259e3c..c21f18d8a 100755 --- a/scripts/clang-format/4.0.1/script.sh +++ b/scripts/clang-format/4.0.1/script.sh @@ -1,5 +1,7 @@ #!/usr/bin/env bash +# For context on this file see https://github.com/mapbox/mason/blob/master/scripts/llvm/base/README.md + # dynamically determine the path to this package HERE="$( cd "$( dirname "${BASH_SOURCE[0]}" )" > /dev/null && pwd )" diff --git a/scripts/clang-tidy/4.0.1/script.sh b/scripts/clang-tidy/4.0.1/script.sh index 391259e3c..c21f18d8a 100755 --- a/scripts/clang-tidy/4.0.1/script.sh +++ b/scripts/clang-tidy/4.0.1/script.sh @@ -1,5 +1,7 @@ #!/usr/bin/env bash +# For context on this file see https://github.com/mapbox/mason/blob/master/scripts/llvm/base/README.md + # dynamically determine the path to this package HERE="$( cd "$( dirname "${BASH_SOURCE[0]}" )" > /dev/null && pwd )" diff --git a/scripts/include-what-you-use/4.0.1/.travis.yml b/scripts/include-what-you-use/4.0.1/.travis.yml new file mode 100644 index 000000000..d22a82784 --- /dev/null +++ b/scripts/include-what-you-use/4.0.1/.travis.yml @@ -0,0 +1,11 @@ +language: generic + +matrix: + include: + - os: osx + - os: linux + sudo: false + +script: +- ./mason build ${MASON_NAME} ${MASON_VERSION} +- ./mason publish ${MASON_NAME} ${MASON_VERSION} \ No newline at end of file diff --git a/scripts/include-what-you-use/4.0.1/script.sh b/scripts/include-what-you-use/4.0.1/script.sh new file mode 100755 index 000000000..c21f18d8a --- /dev/null +++ b/scripts/include-what-you-use/4.0.1/script.sh @@ -0,0 +1,19 @@ +#!/usr/bin/env bash + +# For context on this file see https://github.com/mapbox/mason/blob/master/scripts/llvm/base/README.md + +# dynamically determine the path to this package +HERE="$( cd "$( dirname "${BASH_SOURCE[0]}" )" > /dev/null && pwd )" + +# dynamically take name of package from directory +MASON_NAME=$(basename $(dirname $HERE)) +# dynamically take the version of the package from directory +MASON_VERSION=$(basename $HERE) +MASON_LIB_FILE=bin/${MASON_NAME} + +. ${MASON_DIR}/mason.sh + +# inherit all functions from base +source ${HERE}/../../${MASON_NAME}/base/common.sh + +mason_run "$@" diff --git a/scripts/lldb/4.0.1/.travis.yml b/scripts/lldb/4.0.1/.travis.yml new file mode 100644 index 000000000..d22a82784 --- /dev/null +++ b/scripts/lldb/4.0.1/.travis.yml @@ -0,0 +1,11 @@ +language: generic + +matrix: + include: + - os: osx + - os: linux + sudo: false + +script: +- ./mason build ${MASON_NAME} ${MASON_VERSION} +- ./mason publish ${MASON_NAME} ${MASON_VERSION} \ No newline at end of file diff --git a/scripts/lldb/4.0.1/script.sh b/scripts/lldb/4.0.1/script.sh new file mode 100755 index 000000000..c21f18d8a --- /dev/null +++ b/scripts/lldb/4.0.1/script.sh @@ -0,0 +1,19 @@ +#!/usr/bin/env bash + +# For context on this file see https://github.com/mapbox/mason/blob/master/scripts/llvm/base/README.md + +# dynamically determine the path to this package +HERE="$( cd "$( dirname "${BASH_SOURCE[0]}" )" > /dev/null && pwd )" + +# dynamically take name of package from directory +MASON_NAME=$(basename $(dirname $HERE)) +# dynamically take the version of the package from directory +MASON_VERSION=$(basename $HERE) +MASON_LIB_FILE=bin/${MASON_NAME} + +. ${MASON_DIR}/mason.sh + +# inherit all functions from base +source ${HERE}/../../${MASON_NAME}/base/common.sh + +mason_run "$@" diff --git a/scripts/llvm-cov/4.0.1/.travis.yml b/scripts/llvm-cov/4.0.1/.travis.yml new file mode 100644 index 000000000..d22a82784 --- /dev/null +++ b/scripts/llvm-cov/4.0.1/.travis.yml @@ -0,0 +1,11 @@ +language: generic + +matrix: + include: + - os: osx + - os: linux + sudo: false + +script: +- ./mason build ${MASON_NAME} ${MASON_VERSION} +- ./mason publish ${MASON_NAME} ${MASON_VERSION} \ No newline at end of file diff --git a/scripts/llvm-cov/4.0.1/script.sh b/scripts/llvm-cov/4.0.1/script.sh new file mode 100755 index 000000000..c21f18d8a --- /dev/null +++ b/scripts/llvm-cov/4.0.1/script.sh @@ -0,0 +1,19 @@ +#!/usr/bin/env bash + +# For context on this file see https://github.com/mapbox/mason/blob/master/scripts/llvm/base/README.md + +# dynamically determine the path to this package +HERE="$( cd "$( dirname "${BASH_SOURCE[0]}" )" > /dev/null && pwd )" + +# dynamically take name of package from directory +MASON_NAME=$(basename $(dirname $HERE)) +# dynamically take the version of the package from directory +MASON_VERSION=$(basename $HERE) +MASON_LIB_FILE=bin/${MASON_NAME} + +. ${MASON_DIR}/mason.sh + +# inherit all functions from base +source ${HERE}/../../${MASON_NAME}/base/common.sh + +mason_run "$@" diff --git a/scripts/llvm/4.0.0/script.sh b/scripts/llvm/4.0.0/script.sh index 7bf3cfb29..7a1cd9bd9 100755 --- a/scripts/llvm/4.0.0/script.sh +++ b/scripts/llvm/4.0.0/script.sh @@ -13,11 +13,11 @@ function setup_release() { get_llvm_project "http://llvm.org/releases/${MASON_VERSION}/llvm-${MASON_VERSION}.src.tar.xz" ${MASON_BUILD_PATH}/ c872366ec69fb425ce9336741dba04d9adf16c3e get_llvm_project "http://llvm.org/releases/${MASON_VERSION}/cfe-${MASON_VERSION}.src.tar.xz" ${MASON_BUILD_PATH}/tools/clang 61ca17979e0c2ec7dc5f66cb54ac0b8ec0e653a9 get_llvm_project "http://llvm.org/releases/${MASON_VERSION}/compiler-rt-${MASON_VERSION}.src.tar.xz" ${MASON_BUILD_PATH}/projects/compiler-rt bbf0ad69a296548a44019ed953bdfffc3d3f95d5 - #if [[ ${BUILD_AND_LINK_LIBCXX} == true ]]; then + if [[ ${BUILD_AND_LINK_LIBCXX} == true ]]; then get_llvm_project "http://llvm.org/releases/${MASON_VERSION}/libcxx-${MASON_VERSION}.src.tar.xz" ${MASON_BUILD_PATH}/projects/libcxx ab37f78c83c739d862099a874ad91c3ab295f6b7 get_llvm_project "http://llvm.org/releases/${MASON_VERSION}/libcxxabi-${MASON_VERSION}.src.tar.xz" ${MASON_BUILD_PATH}/projects/libcxxabi d36cb0be6c8fa54534a1cb067bdb4a6b3389d3d3 get_llvm_project "http://llvm.org/releases/${MASON_VERSION}/libunwind-${MASON_VERSION}.src.tar.xz" ${MASON_BUILD_PATH}/projects/libunwind 5178ea6b1834a7cd3d6be610709b811255255a11 - #fi + fi get_llvm_project "http://llvm.org/releases/${MASON_VERSION}/lld-${MASON_VERSION}.src.tar.xz" ${MASON_BUILD_PATH}/tools/lld 137e8c9d5d73a0e3dfec1ffd63abb5173473e95b get_llvm_project "http://llvm.org/releases/${MASON_VERSION}/clang-tools-extra-${MASON_VERSION}.src.tar.xz" ${MASON_BUILD_PATH}/tools/clang/tools/extra 9c9ad20062c41f22f4078dcfea38daebfce2c2ed get_llvm_project "http://llvm.org/releases/${MASON_VERSION}/lldb-${MASON_VERSION}.src.tar.xz" ${MASON_BUILD_PATH}/tools/lldb 5b17f041631670e40718ab89f99f1c8347766d92 diff --git a/scripts/llvm/4.0.1/README.md b/scripts/llvm/4.0.1/README.md new file mode 100644 index 000000000..425bf0d1e --- /dev/null +++ b/scripts/llvm/4.0.1/README.md @@ -0,0 +1 @@ +For context on this file see https://github.com/mapbox/mason/blob/master/scripts/llvm/base/README.md diff --git a/scripts/llvm/4.0.1/script.sh b/scripts/llvm/4.0.1/script.sh index 103464b28..f97b4955f 100755 --- a/scripts/llvm/4.0.1/script.sh +++ b/scripts/llvm/4.0.1/script.sh @@ -1,5 +1,7 @@ #!/usr/bin/env bash +# For context on this file see https://github.com/mapbox/mason/blob/master/scripts/llvm/base/README.md + # dynamically determine the path to this package HERE="$( cd "$( dirname "${BASH_SOURCE[0]}" )" > /dev/null && pwd )" # dynamically take name of package from directory @@ -10,19 +12,22 @@ MASON_VERSION=$(basename $HERE) source ${HERE}/../../${MASON_NAME}/base/common.sh function setup_release() { - get_llvm_project "http://llvm.org/releases/${MASON_VERSION}/llvm-${MASON_VERSION}.src.tar.xz" ${MASON_BUILD_PATH}/ 4b9869229fb4fd552a4cf2c9a1907ae5abf34d8f - get_llvm_project "http://llvm.org/releases/${MASON_VERSION}/cfe-${MASON_VERSION}.src.tar.xz" ${MASON_BUILD_PATH}/tools/clang 10cbfde24b237cf2a574de2af89f6f02ebb5a40c - get_llvm_project "http://llvm.org/releases/${MASON_VERSION}/compiler-rt-${MASON_VERSION}.src.tar.xz" ${MASON_BUILD_PATH}/projects/compiler-rt 4ff8f64d4e37438a098c598cc19ec47a9c3b8f96 - #if [[ ${BUILD_AND_LINK_LIBCXX} == true ]]; then - get_llvm_project "http://llvm.org/releases/${MASON_VERSION}/libcxx-${MASON_VERSION}.src.tar.xz" ${MASON_BUILD_PATH}/projects/libcxx 307a5eb570c572eee673fea68848cd64b060b268 - get_llvm_project "http://llvm.org/releases/${MASON_VERSION}/libcxxabi-${MASON_VERSION}.src.tar.xz" ${MASON_BUILD_PATH}/projects/libcxxabi adae1419c61953b5b1008a734c65e23d66315caf - get_llvm_project "http://llvm.org/releases/${MASON_VERSION}/libunwind-${MASON_VERSION}.src.tar.xz" ${MASON_BUILD_PATH}/projects/libunwind f48776ac202cc9ceb9bf277670ac1e0e4e26d03d - #fi - get_llvm_project "http://llvm.org/releases/${MASON_VERSION}/lld-${MASON_VERSION}.src.tar.xz" ${MASON_BUILD_PATH}/tools/lld d25b6d361ac76798e1a3c53ad1325ee038e2b5a8 - get_llvm_project "http://llvm.org/releases/${MASON_VERSION}/clang-tools-extra-${MASON_VERSION}.src.tar.xz" ${MASON_BUILD_PATH}/tools/clang/tools/extra aa19783e2cd246aab8ae8c2826f81e2c500b6435 - get_llvm_project "http://llvm.org/releases/${MASON_VERSION}/lldb-${MASON_VERSION}.src.tar.xz" ${MASON_BUILD_PATH}/tools/lldb d711485022e7c13f92673dd2873d756d0c562faa - #get_llvm_project "https://github.com/include-what-you-use/include-what-you-use/archive/clang_${MAJOR_MINOR}.tar.gz" ${MASON_BUILD_PATH}/tools/clang/tools/include-what-you-use - get_llvm_project "https://github.com/include-what-you-use/include-what-you-use.git" ${MASON_BUILD_PATH}/tools/clang/tools/include-what-you-use + # usage: get_llvm_project [url or git url] [path to download to] + get_llvm_project "http://llvm.org/releases/${MASON_VERSION}/llvm-${MASON_VERSION}.src.tar.xz" ${MASON_BUILD_PATH}/ + get_llvm_project "http://llvm.org/releases/${MASON_VERSION}/cfe-${MASON_VERSION}.src.tar.xz" ${MASON_BUILD_PATH}/tools/clang + get_llvm_project "http://llvm.org/releases/${MASON_VERSION}/compiler-rt-${MASON_VERSION}.src.tar.xz" ${MASON_BUILD_PATH}/projects/compiler-rt + if [[ ${BUILD_AND_LINK_LIBCXX} == true ]]; then + get_llvm_project "http://llvm.org/releases/${MASON_VERSION}/libcxx-${MASON_VERSION}.src.tar.xz" ${MASON_BUILD_PATH}/projects/libcxx + get_llvm_project "http://llvm.org/releases/${MASON_VERSION}/libcxxabi-${MASON_VERSION}.src.tar.xz" ${MASON_BUILD_PATH}/projects/libcxxabi + get_llvm_project "http://llvm.org/releases/${MASON_VERSION}/libunwind-${MASON_VERSION}.src.tar.xz" ${MASON_BUILD_PATH}/projects/libunwind + fi + get_llvm_project "http://llvm.org/releases/${MASON_VERSION}/lld-${MASON_VERSION}.src.tar.xz" ${MASON_BUILD_PATH}/tools/lld + get_llvm_project "http://llvm.org/releases/${MASON_VERSION}/clang-tools-extra-${MASON_VERSION}.src.tar.xz" ${MASON_BUILD_PATH}/tools/clang/tools/extra + get_llvm_project "http://llvm.org/releases/${MASON_VERSION}/lldb-${MASON_VERSION}.src.tar.xz" ${MASON_BUILD_PATH}/tools/lldb + get_llvm_project "https://github.com/include-what-you-use/include-what-you-use/archive/clang_${MAJOR_MINOR}.tar.gz" ${MASON_BUILD_PATH}/tools/clang/tools/include-what-you-use + # Note: the below commented line uses a custom gitsha to pin to a working commit if the include-what-you-use team has not yet created a tag for the given clang major version + # This happended previously a https://github.com/include-what-you-use/include-what-you-use/issues/397#issuecomment-313479507 + #get_llvm_project "https://github.com/include-what-you-use/include-what-you-use.git" ${MASON_BUILD_PATH}/tools/clang/tools/include-what-you-use "" 45e1264507f5e2725289ca3a0f4de98108e964c7 } mason_run "$@" diff --git a/scripts/llvm/base/README.md b/scripts/llvm/base/README.md index 4bd90d815..f51eb38cf 100644 --- a/scripts/llvm/base/README.md +++ b/scripts/llvm/base/README.md @@ -1,40 +1,205 @@ -## llvm +# llvm -This is a package of llvm. Multiple packages are based on repackaging the binaries from this one. +This readme documents: -### Depends + - How the llvm packages are structured + - Which sub-packages depend on the llvm package + - How the llvm packages are built + - How to create a new llvm package + sub-packages + - How to use the binary packages -#### On Linux +## How the llvm packages are structured - - >= Ubuntu precise - - binutils 2.27 from mason for -flto support +This llvm/base directory is not a package itself, but two critical building blocks: -#### On OS X + - `scripts/llvm/base/README.md`: this document, that explains how to understand, use, and build the llvm package and sub-packages + - `scripts/llvm/base/common.sh`: a common set of bash functions that are inherited by each llvm package. - - OS X >= 10.11 - - On XCode >= 8 installed to `/Applications/Xcode.app` such that `xcode-select -p` returns `/Applications/Xcode.app/Contents/Developer` +A given llvm package version is what users of mason actually request: e.g. llvm 4.0.0. This package lives at `scripts/llvm/4.0.0/script.sh`. -If you don't have Xcode installed in `/Applications` you can still use the clang++ in this package but you need to set: +A user can install that version like: -1) `-isysroot` to point at your Mac SDK (the value returned from `xcrun --show-sdk-path`) +``` +mason install llvm 4.0.0 +``` -2) Add an include flag to point at your C++ headers which are generally at `ls $(dirname $(xcrun --find clang++))/../include/c++/v1`. For the command line tools this directory is at `/Library/Developer/CommandLineTools/usr/include/c++/v1/` +However, if they do they will get over 1.6 GB of binaries. See below to learn about the subpackages. -### Details of builds +## Which sub-packages depend on the llvm package - - OS X build done on OS X 10.12.1 - - Linux build done on Ubuntu precise +To enable faster and smaller installs, we provide sub-packages that provide specific working parts of llvm. The available sub-packages are: -Builds (and rebuilds) are done using `utils/rebuild-llvm-tools.sh` + - `clang++`: The clang++ (C++) and clang (C) compiler in one package - https://clang.llvm.org. Also includes: + - https://compiler-rt.llvm.org/ for sanitizer support + - `libLTO` for link time optimization support - https://www.llvm.org/docs/LinkTimeOptimization.html#liblto (needs binutils >= 2.27) + - `clang-format`: command line tool to do automatic source code formatting based on Clang - https://clang.llvm.org/docs/ClangFormat.html + - `clang-tidy`: a clang-based C++ “linter” tool. Its purpose is to provide an extensible framework for diagnosing and fixing typical programming errors, like style violations, interface misuse, or bugs that can be deduced via static analysis - https://clang.llvm.org/extra/clang-tidy + - `include-what-you-use`: A tool for use with clang to analyze #includes in C and C++ source files - https://include-what-you-use.org + - `lldb`: LLDB is a next generation, high-performance debugger - https://lldb.llvm.org + - `llvm-cov`: -#### OS X details +These subpackages use custom bash scripts that also inherit from a `base/common.sh`. For example for the `clang++` sub-package this is `scripts/clang++/base/common.sh`. This design makes it possible to create new versions of the packages, by copying a specific verison, without modifying needing to modify the `script.sh` (the version is automatically detected from their directory name). -On MacOS we use the apple system clang++ to compile and link against the apple system libc++. -#### Linux details +## How the llvm packages are built -On Linux we use Ubuntu Precise in order to ensure that binaries can be run on Ubuntu Precise and newer platforms. +They are built locally by a Mason contributor, rather than on https://travis-ci.org. The reason is that travis is not fast enough for the llvm compile. + +### OS X details + +On MacOS we build for OS X 10.12 as the minimum target. We link llvm against the apple system libc++. + +### Linux details + +On Linux we build using an Ubuntu Precise docker image in order to ensure that binaries can be run on Ubuntu Precise and newer platforms. We link clang++ and other llvm c++ tools to libc++ (the one within the build) to avoid clang++ itself depending on a specific libstdc++ version. -The clang++ binary still defaults to building and linking linux programs against libstdc++. \ No newline at end of file +The clang++ binary still defaults to building and linking linux programs against libstdc++. + + +## How to create a new llvm package + sub-packages + +### Pre-requisites + +If you would like to create a new llvm package, when a new version is created, you will need: + + - Docker installed + - At least 1 hour free to run builds locally + - A fast internet connection for large uploads (> 1.6 GB) + - A host machine of either Linux or OS X + +Here are the steps to create a new llvm version: + +#### Step 1: Note the latest release + +Go to http://releases.llvm.org to see the latest release. For our walkthrough below we will assume that the new release is `4.0.2`. + +#### Step 2: Make sure this release of llvm is not already packaged + +Check to see if there is not already a `scripts/llvm/4.0.2`. + +#### Step 3: Create a mason branch + +Create a new branch called `llvm-4.0.2`. + +#### Step 4: Create the new package + +Top create new version of the llvm package and sub-packages do: + +``` +./utils/llvm.sh create 4.0.2 4.0.1 +``` + +#### Step 5: Push packages to github and create PR + +First add the new package and sub-packages to git + +``` +git add scripts/*/4.0.2/* +``` + +Now push to github: + +``` +git push origin llvm-4.0.2 +``` + +Then go create a PR. + +#### Step 6: Build and publish the new llvm package and sub-packages + +This step will vary depending on your host operating system. + +If you are on OS X, then you will first build the package locally to produce the OS X version. Then you will build the package a second time inside a linux docker container to produce the linux version. + +If you are on Linux, then you will first build the package in the linux docker container. Then you will ping a co-worker to help build the package on OS X if you do not have access to an OS X machine. + +##### OS X + +A. First build the llvm package. This may take > 30 minutes. + +``` +./mason build llvm 4.0.2 +``` + +B. Then build all the sub-packages. This will be fast since they are simply repackaged binaries. + +``` +./utils/llvm.sh build 4.0.2 +``` + +C. Authenticate your shell with the mason AWS KEYS + +``` +export AWS_ACCESS_KEY_ID=<> +export AWS_SECRET_ACCESS_KEY=<> +``` + +D. Then publish the main llvm package and sub-packages. This may take > 20 min depending on the speed of your internet connection. + +``` +./mason publish llvm 4.0.2 +./utils/llvm.sh publish 4.0.2 +``` + +##### Linux + +A. Build the docker image + +``` +docker build -t mason-llvm -f utils/Dockerfile.llvm . +``` + +B. Run the docker image + +We run the docker image to build the package on linux. We map volumes such that the binary will end up on our host machine (to avoid needing to pass publishing credentials to docker). + +``` +LLVM_VERSION="4.0.2" +docker run -it --volume $(pwd):/home/travis/build/mapbox/mason mason-llvm \ + ./mason build llvm ${LLVM_VERSION} && ./utils/llvm.sh build ${LLVM_VERSION} +``` + +C. Authenticate your shell with the mason AWS KEYS + +``` +export AWS_ACCESS_KEY_ID=<> +export AWS_SECRET_ACCESS_KEY=<> +``` + +D. Then publish the main llvm package and sub-packages. This may take > 20 min depending on the speed of your internet connection. + +``` +MASON_PLATFORM=linux ./mason publish llvm 4.0.2 +MASON_PLATFORM=linux ./utils/llvm.sh publish 4.0.2 +``` + +Note: `MASON_PLATFORM=linux` is only needed if your host is OS X. + +## How to use the binary packages + +The binary packages will work on: + +- >= Ubuntu precise +- OS X >= 10.12 +- XCode >= 8 installed to `/Applications/Xcode.app` such that `xcode-select -p` returns `/Applications/Xcode.app/Contents/Developer` + +### LTO support + +If you want to use `-flto` support with clang++ you also need to install binutils >= 2.27. To see the exact version of binutils that llvm was built against look inside the `scripts/llvm/base/common.sh` (https://github.com/mapbox/mason/blob/llvm-4.0.1/scripts/llvm/base/common.sh#L127). + +### Xcode missing + +It is recommended that you install XCode >= 8. In the rare situation that you cannot, or when you don't have Xcode installed in `/Applications`, you may be able to get clang++ working by: + +1) Adding `-isysroot` compiler argument to point at your Mac SDK (the value returned from `xcrun --show-sdk-path`) + +2) Adding an include flag to point at your C++ headers which are generally at `ls $(dirname $(xcrun --find clang++))/../include/c++/v1`. For the command line tools this directory is at `/Library/Developer/CommandLineTools/usr/include/c++/v1/` + +For `make` based systems this might work: + +``` +export CXXFLAGS="-isysroot $(xcrun --show-sdk-path) -I$(dirname $(xcrun --find clang++))/../include/c++/v1" +``` + diff --git a/scripts/llvm/base/common.sh b/scripts/llvm/base/common.sh index 273af3ab3..6bbfe2070 100755 --- a/scripts/llvm/base/common.sh +++ b/scripts/llvm/base/common.sh @@ -24,6 +24,7 @@ else fi # we use this custom function rather than "mason_download" since we need to easily grab multiple packages +# get_llvm_project [url or git url] [path to download to] function get_llvm_project() { local URL=${1} local TO_DIR=${2} @@ -32,16 +33,24 @@ function get_llvm_project() { exit 1 fi local EXPECTED_HASH=${3:-false} + local CUSTOM_GITSHA=${4:-false} + local DEPTH=1 + if [[ ${CUSTOM_GITSHA:-false} == false ]]; then + DEPTH=500 + fi local file_basename=$(basename ${URL}) local local_file_or_checkout=$(pwd)/${file_basename} if [[ ${URL} =~ '.git' ]]; then if [ ! -d ${local_file_or_checkout} ] ; then mason_step "cloning ${URL} to ${local_file_or_checkout}" - git clone --depth 1 ${URL} ${local_file_or_checkout} + git clone --depth ${DEPTH} ${URL} ${local_file_or_checkout} else mason_substep "already cloned ${URL}, pulling to update" (cd ${local_file_or_checkout} && git pull) fi + if [[ ${CUSTOM_GITSHA:-false} == false ]]; then + (cd ${local_file_or_checkout} && git fetch && git checkout ${CUSTOM_GITSHA}) + fi mason_step "moving ${local_file_or_checkout} into place at ${TO_DIR}" cp -r ${local_file_or_checkout} ${TO_DIR} else @@ -53,7 +62,7 @@ function get_llvm_project() { fi export OBJECT_HASH=$(git hash-object ${local_file_or_checkout}) if [[ ${EXPECTED_HASH:-false} == false ]]; then - mason_error "Warning: no expected hash provided by script.sh, actual was ${OBJECT_HASH}" + mason_error "NOTICE: detected object has of ${OBJECT_HASH}, optionally add this the 'setup_release' function in your script.sh in order to assert this never changes" else if [[ $3 != ${OBJECT_HASH} ]]; then mason_error "Error: hash mismatch ${EXPECTED_HASH} (expected) != ${OBJECT_HASH} (actual)" @@ -103,7 +112,7 @@ function mason_prepare_compile { CCACHE_VERSION=3.3.1 CMAKE_VERSION=3.7.2 NINJA_VERSION=1.7.1 - CLANG_VERSION=3.9.1 + CLANG_VERSION=4.0.0 ${MASON_DIR}/mason install clang++ ${CLANG_VERSION} MASON_CLANG=$(${MASON_DIR}/mason prefix clang++ ${CLANG_VERSION}) @@ -177,7 +186,7 @@ function mason_compile { # because that value will be appended to the sysroot, not exist, and then get thrown out. If the sysroot were / then it would be added CMAKE_EXTRA_ARGS="${CMAKE_EXTRA_ARGS} -DDEFAULT_SYSROOT=/Applications/Xcode.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/MacOSX.sdk" CMAKE_EXTRA_ARGS="${CMAKE_EXTRA_ARGS} -DCLANG_DEFAULT_CXX_STDLIB=libc++" - CMAKE_EXTRA_ARGS="${CMAKE_EXTRA_ARGS} -DCMAKE_OSX_DEPLOYMENT_TARGET=10.11" + CMAKE_EXTRA_ARGS="${CMAKE_EXTRA_ARGS} -DCMAKE_OSX_DEPLOYMENT_TARGET=10.12" CMAKE_EXTRA_ARGS="${CMAKE_EXTRA_ARGS} -DLLVM_CREATE_XCODE_TOOLCHAIN=ON -DLLVM_EXTERNALIZE_DEBUGINFO=ON" fi if [[ $(uname -s) == 'Linux' ]]; then @@ -188,7 +197,7 @@ function mason_compile { fi fi - # we strip this since we don't care about older os x for this package + # Strip this since we set CMAKE_OSX_DEPLOYMENT_TARGET above. We assume that we'd only upgrade to use this compiler on recent OS X systems and we want the potential performance benefit of targeting a more recent version if [[ $(uname -s) == 'Darwin' ]]; then export CXXFLAGS="${CXXFLAGS//-mmacosx-version-min=10.8}" export LDFLAGS="${LDFLAGS//-mmacosx-version-min=10.8}" diff --git a/utils/Dockerfile.llvm b/utils/Dockerfile.llvm new file mode 100644 index 000000000..970fa2874 --- /dev/null +++ b/utils/Dockerfile.llvm @@ -0,0 +1,28 @@ +FROM ubuntu:12.04 + +# Used to build llvm and sub-packages +# See https://github.com/mapbox/mason/blob/master/scripts/llvm/base/README.md for details +# docker build -t mason-llvm -f utils/Dockerfile.llvm +# docker run -it mason-llvm bash + +# we mimic the travis path to ensure any hardcoded paths in deps +# do not cause problem's when building (an example is freetype-config) +ENV WORKINGDIR /home/travis/build/mapbox/mason/ +WORKDIR ${WORKINGDIR} + +COPY mason mason +COPY mason.sh mason.sh +COPY scripts scripts +COPY utils utils + +RUN apt-get update -y && \ + apt-get install -y vim python build-essential bash curl git-core ca-certificates software-properties-common python-software-properties --no-install-recommends + +# Note: we add the ubuntu-toolchain-r PPA to be able to upgrade to libstdc++6 below +# which is a runtime dependency of the build tools for llvm like cmake. +# We do not actually link or use libstdc++ for llvm tools (rather, on linux, they link their own libc++) +RUN add-apt-repository -y ppa:ubuntu-toolchain-r/test && \ + apt-get update -y + +# curses needed until https://github.com/mapbox/mason/issues/309 is solved. +RUN apt-get install -y libstdc++6 xutils-dev libncurses5-dev libz-dev diff --git a/utils/llvm.sh b/utils/llvm.sh new file mode 100755 index 000000000..d45b41091 --- /dev/null +++ b/utils/llvm.sh @@ -0,0 +1,99 @@ +#!/usr/bin/env bash + +set -eu +set -o pipefail + +: ' + +For more details on how to use this script see https://github.com/mapbox/mason/blob/master/scripts/llvm/base/README.md + +' + +function usage() { + echo "Usage workflow:" + echo + echo " Create a new llvm package and sub-packages:" + echo " ./utils/llvm.sh create " + echo + echo " Test building new llvm package:" + echo " ./mason build llvm " + echo + echo " Build the new llvm sub-packages:" + echo " ./utils/llvm.sh build " + echo + echo " Publish the llvm package:" + echo " ./mason publish llvm " + echo + echo " Publish all the sub-packages:" + echo " ./utils/llvm.sh publish " +} + +subpackages=(clang++ clang-tidy clang-format lldb llvm-cov include-what-you-use) + +function build() { + local VERSION=$1 + for package in "${!subpackages[@]}"; do + ./mason build ${subpackages[$package]} ${VERSION} + done +} + +function publish() { + local VERSION=$1 + for package in "${!subpackages[@]}"; do + ./mason publish ${subpackages[$package]} ${VERSION} + done +} + +function create() { + if [[ ! ${1:-} ]]; then + usage + echo + echo + echo "ERROR: please provide first arg of new version" + exit 1 + fi + if [[ -d ./scripts/llvm/${1} ]]; then + usage + echo + echo + echo "ERROR: first arg must point to a version of llvm that does not exist" + exit 1 + fi + if [[ ! -d ./scripts/llvm/${2} ]]; then + usage + echo + echo + echo "ERROR: second arg must point to a version of llvm that already exists (since we need to copy from it)" + exit 1 + fi + if [[ ! ${2:-} ]]; then + usage + echo + echo + echo "ERROR: please provide second arg of version to copy from" + exit 1 + fi + + local NEW_VERSION="$1" + local LAST_VERSION="$2" + mkdir -p scripts/llvm/${NEW_VERSION} + cp -r scripts/llvm/${LAST_VERSION}/. scripts/llvm/${NEW_VERSION}/ + for package in "${!subpackages[@]}"; do + mkdir -p scripts/${subpackages[$package]}/${NEW_VERSION} + cp -r scripts/${subpackages[$package]}/${LAST_VERSION}/. scripts/${subpackages[$package]}/${NEW_VERSION}/ + done +} + +if [[ ${1:-0} == "create" ]]; then + shift + create $@ +elif [[ ${1:-0} == "build" ]]; then + shift + build $@ +elif [[ ${1:-0} == "publish" ]]; then + shift + publish $@ +else + usage + exit 1 +fi diff --git a/utils/rebuild-llvm-tools.sh b/utils/rebuild-llvm-tools.sh deleted file mode 100755 index 28ed04043..000000000 --- a/utils/rebuild-llvm-tools.sh +++ /dev/null @@ -1,51 +0,0 @@ -#!/usr/bin/env bash - -set -eu -set -o pipefail - -packages=(llvm clang++ clang-tidy clang-format lldb llvm-cov include-what-you-use) -versions=(3.9.1 4.0.0 3.8.1) - -if [[ $(uname -s) == 'Linux' ]]; then - CLANG_BOOTSTRAP_VERSION="3.8.1" - ./mason install clang++ ${CLANG_BOOTSTRAP_VERSION} - CLANG_PREFIX=$(./mason prefix clang++ ${CLANG_BOOTSTRAP_VERSION}) - export CXX=${CLANG_PREFIX}/bin/clang++ - export CC=${CLANG_PREFIX}/bin/clang -fi - -function build() { - local VERSION=$1 - for package in "${!packages[@]}"; do - ./mason build ${packages[$package]} ${VERSION} - done -} - -function publish() { - local VERSION=$1 - for package in "${!packages[@]}"; do - ./mason publish ${packages[$package]} ${VERSION} - done -} - -# to be used by sourcing and running manually -function new_version() { - local NEW_VERSION="$1" - local LAST_VERSION="$2" - for package in "${!packages[@]}"; do - mkdir -p scripts/${packages[$package]}/${NEW_VERSION} - cp -r scripts/${packages[$package]}/${LAST_VERSION}/. scripts/${packages[$package]}/${NEW_VERSION}/ - done -} - - -function build_all() { - for ver in "${!versions[@]}"; do - build ${versions[$ver]} - publish ${versions[$ver]} - done -} - -if [[ ${1:-0} == "all" ]]; then - build_all -fi