From 75dee8f21e4a0adbf1ea6ba835a537ab640a65e2 Mon Sep 17 00:00:00 2001 From: Dane Springmeyer Date: Sat, 9 Sep 2017 17:13:13 -0700 Subject: [PATCH 1/9] add back ability to build against external mapnik - refs #227 --- Makefile | 40 +++++++++++++++++++++++++++++----------- README.md | 39 +++++++++++++++++---------------------- bootstrap.sh | 33 ++++++++++++++++++++------------- gyp/build.gyp | 29 ++++++++++++++++------------- install_mason.sh | 14 ++++++++------ 5 files changed, 90 insertions(+), 65 deletions(-) diff --git a/Makefile b/Makefile index 90b50e11..580e064b 100755 --- a/Makefile +++ b/Makefile @@ -1,27 +1,43 @@ -MAPNIK_PLUGINDIR = $(shell mason_packages/.link/bin/mapnik-config --input-plugins) -BUILDTYPE ?= Release - GYP_REVISION=3464008 -all: libvtile +default: release -mason_packages/.link/bin/mapnik-config: +mason_packages/.link/bin: ./install_mason.sh +mason_packages/.link/bin/mapnik-config: + SKIP_MAPNIK_INSTALL=YES ./install_mason.sh + +pre_build_check: + @echo "Looking for mapnik-config on your PATH..." + mapnik-config -v + ./deps/gyp: git clone https://chromium.googlesource.com/external/gyp.git ./deps/gyp && cd ./deps/gyp && git checkout $(GYP_REVISION) -build/Makefile: mason_packages/.link/bin/mapnik-config ./deps/gyp gyp/build.gyp test/* - deps/gyp/gyp gyp/build.gyp --depth=. -DMAPNIK_PLUGINDIR=\"$(MAPNIK_PLUGINDIR)\" -Goutput_dir=. --generator-output=./build -f make +build/Makefile: ./deps/gyp gyp/build.gyp test/* + deps/gyp/gyp gyp/build.gyp --depth=. -DMAPNIK_PLUGINDIR=\"$(shell mapnik-config --input-plugins)\" -Goutput_dir=. --generator-output=./build -f make -libvtile: build/Makefile Makefile - PATH="`pwd`/mason_packages/.link/bin/:${PATH}" $(MAKE) -C build/ BUILDTYPE=$(BUILDTYPE) V=$(V) +release: mason_packages/.link/bin/mapnik-config build/Makefile Makefile + CXXFLAGS="-D_GLIBCXX_USE_CXX11_ABI=0" PATH="`pwd`/mason_packages/.link/bin/:${PATH}" $(MAKE) -C build/ BUILDTYPE=Release V=$(V) + +debug: mason_packages/.link/bin/mapnik-config build/Makefile Makefile + CXXFLAGS="-D_GLIBCXX_USE_CXX11_ABI=0" PATH="`pwd`/mason_packages/.link/bin/:${PATH}" $(MAKE) -C build/ BUILDTYPE=Debug V=$(V) + +release_base: pre_build_check mason_packages/.link/bin build/Makefile Makefile + $(MAKE) -C build/ BUILDTYPE=Release V=$(V) + +debug_base: pre_build_check mason_packages/.link/bin build/Makefile Makefile + $(MAKE) -C build/ BUILDTYPE=Debug V=$(V) test/geometry-test-data/README.md: git submodule update --init -test: libvtile test/geometry-test-data/README.md - BUILDTYPE=$(BUILDTYPE) ./test/run.sh +test: test/geometry-test-data/README.md + BUILDTYPE=Release ./test/run.sh + +test-debug: test/geometry-test-data/README.md + BUILDTYPE=Debug ./test/run.sh testpack: rm -f ./*tgz @@ -31,6 +47,8 @@ testpack: clean: rm -rf ./build + +distclean: clean rm -rf ./mason_packages .PHONY: test diff --git a/README.md b/README.md index 3baf062e..5e8354a7 100644 --- a/README.md +++ b/README.md @@ -11,42 +11,37 @@ Provides C++ headers that support rendering geodata into vector tiles and render ## Depends -Mapnik Vector Tile does not currently support Mapnik 3.1.x. - - - mapnik-vector-tile >=1.4.x depends on Mapnik >=v3.0.14 - - mapnik-vector-tile >=1.0.x depends on Mapnik >=v3.0.11 - - mapnik-vector-tile 1.0.0 to 0.7.x depends on Mapnik v3.0.x (until 3.0.0 is released this means latest mapnik HEAD) - - mapnik-vector-tile 0.6.x and previous work with Mapnik v2.2.x or v2.3.x - - You will need `libmapnik` and `mapnik-config` available - - Protobuf: `libprotobuf` and `protoc` - ## Implementation details Vector tiles in this code represent a direct serialization of Mapnik layers optimized for space efficient storage and fast deserialization. For those familiar with the Mapnik API vector tiles here can be considered a named array of `mapnik::featureset_ptr` whose geometries have been pre-tiled. For more details see [vector-tile-spec](https://github.com/mapbox/vector-tile-spec). -### Ubuntu Dependencies Installation +## Building from source + +If you do not need to build against an external mapnik, just type: - sudo apt-get install -y libprotobuf7 libprotobuf-dev protobuf-compiler - sudo apt-add-repository --yes ppa:mapnik/nightly-2.3 - sudo apt-get update -y - sudo apt-get -y install libmapnik=2.3.0* mapnik-utils=2.3.0* libmapnik-dev=2.3.0* mapnik-input-plugin*=2.3.0* + make -### OS X Dependencies Installation +This will download all deps (including Mapnik) and compile against them. - brew install protobuf - brew install mapnik +To build and test in debug mode do: -## Building + make debug test-debug -Just type: +If you have Mapnik, libprotobuf, and all the Mapnik deps already installed on your system then you can build against them with: - make + make release_base + +If building against an external Mapnik please know that Mapnik Vector Tile does not currently support Mapnik 3.1.x. -This builds the protobuf C++ wrappers: `vector_tile.pb.cc` and `vector_tile.pb.h` + - mapnik-vector-tile >=1.4.x depends on Mapnik >=v3.0.14 + - mapnik-vector-tile >=1.0.x depends on Mapnik >=v3.0.11 + - mapnik-vector-tile 1.0.0 to 0.7.x depends on Mapnik v3.0.x (until 3.0.0 is released this means latest mapnik HEAD) + - mapnik-vector-tile 0.6.x and previous work with Mapnik v2.2.x or v2.3.x + - You will need `libmapnik` and `mapnik-config` available + - Protobuf: `libprotobuf` and `protoc` -Then include `vector_tile.pb.cc` in your code. The rest is header only. ## Tests diff --git a/bootstrap.sh b/bootstrap.sh index f47e3850..bf125c46 100755 --- a/bootstrap.sh +++ b/bootstrap.sh @@ -1,20 +1,27 @@ #!/usr/bin/env bash +# if built against mason package fix dynamic data locations function setup_runtime_settings() { - local MASON_LINKED_ABS=$(pwd)/mason_packages/.link - export PROJ_LIB=${MASON_LINKED_ABS}/share/proj - ICU_VERSION=$(ls ${MASON_LINKED_ABS}/share/icu/) - export ICU_DATA=${MASON_LINKED_ABS}/share/icu/${ICU_VERSION} - export GDAL_DATA=${MASON_LINKED_ABS}/share/gdal - if [[ $(uname -s) == 'Darwin' ]]; then - export DYLD_LIBRARY_PATH=$(pwd)/mason_packages/.link/lib:${DYLD_LIBRARY_PATH:-} - # OS X > 10.11 blocks DYLD_LIBRARY_PATH so we pass along using a - # differently named variable - export MVT_LIBRARY_PATH=${DYLD_LIBRARY_PATH} - else - export LD_LIBRARY_PATH=$(pwd)/mason_packages/.link/lib:${LD_LIBRARY_PATH:-} + if [[ -f $(pwd)/mason_packages/.link/bin/mapnik-config ]]; then + # TODO: use --proj-lib, --gdal-data, etc after https://github.com/mapnik/mapnik/pull/3759 is fixed + #export PROJ_LIB=$(mapnik-config --proj-lib) + #export GDAL_DATA=$(mapnik-config --gdal-data) + #export ICU_DATA=$(mapnik-config --icu-data) + local MASON_LINKED_ABS=$(pwd)/mason_packages/.link + export PROJ_LIB=${MASON_LINKED_ABS}/share/proj + ICU_VERSION=$(ls ${MASON_LINKED_ABS}/share/icu/) + export ICU_DATA=${MASON_LINKED_ABS}/share/icu/${ICU_VERSION} + export GDAL_DATA=${MASON_LINKED_ABS}/share/gdal + if [[ $(uname -s) == 'Darwin' ]]; then + export DYLD_LIBRARY_PATH=$(pwd)/mason_packages/.link/lib:${DYLD_LIBRARY_PATH:-} + # OS X > 10.11 blocks DYLD_LIBRARY_PATH so we pass along using a + # differently named variable + export MVT_LIBRARY_PATH=${DYLD_LIBRARY_PATH} + else + export LD_LIBRARY_PATH=$(pwd)/mason_packages/.link/lib:${LD_LIBRARY_PATH:-} + fi + export PATH=$(pwd)/mason_packages/.link/bin:${PATH} fi - export PATH=$(pwd)/mason_packages/.link/bin:${PATH} } function main() { diff --git a/gyp/build.gyp b/gyp/build.gyp index 5a22717d..611c443e 100644 --- a/gyp/build.gyp +++ b/gyp/build.gyp @@ -38,17 +38,18 @@ ], 'include_dirs': [ '<(SHARED_INTERMEDIATE_DIR)/', + '../mason_packages/.link/include/' ], 'cflags_cc' : [ '-D_THREAD_SAFE', - ' Date: Sat, 9 Sep 2017 17:16:34 -0700 Subject: [PATCH 2/9] fix undefined varg --- install_mason.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/install_mason.sh b/install_mason.sh index c4854b93..192b304b 100755 --- a/install_mason.sh +++ b/install_mason.sh @@ -22,7 +22,7 @@ install wagyu 0.4.2 install protobuf 2.6.1 # mapnik -if [[ ! ${SKIP_MAPNIK_INSTALL} ]] && [[ ! -f ./mason_packages/.link/bin/mapnik-config ]]; then +if [[ ! ${SKIP_MAPNIK_INSTALL:-true} ]] && [[ ! -f ./mason_packages/.link/bin/mapnik-config ]]; then # mapnik deps install jpeg_turbo 1.5.1 From 708b54e15a9de256f55e2db67f1601bda5b9d2ef Mon Sep 17 00:00:00 2001 From: Dane Springmeyer Date: Sat, 9 Sep 2017 17:28:19 -0700 Subject: [PATCH 3/9] upgrade deps, fix build --- .travis.yml | 10 +++++----- install_mason.sh | 11 ++++++----- scripts/build.sh | 9 +++++++-- 3 files changed, 18 insertions(+), 12 deletions(-) diff --git a/.travis.yml b/.travis.yml index c359ee9b..1c442d2c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -33,7 +33,7 @@ matrix: - os: linux sudo: false compiler: ": clang-coverage" - env: JOBS=10 CXX="ccache clang++-3.9 -Qunused-arguments" COVERAGE=true BUILDTYPE=Debug + env: JOBS=10 CXX="ccache clang++-3.9 -Qunused-arguments" COVERAGE=true addons: apt: sources: ['ubuntu-toolchain-r-test'] @@ -48,11 +48,11 @@ before_install: - ./install_mason.sh - | if [[ $(uname -s) == 'Linux' ]]; then - ./mason/mason install llvm-cov 3.9.1 - ./mason/mason install clang++ 3.9.1 - export PATH=$(./mason/mason prefix clang++ 3.9.1)/bin:${PATH} + ./mason/mason install llvm-cov 4.0.1 + ./mason/mason install clang++ 4.0.1 + export PATH=$(./mason/mason prefix clang++ 4.0.1)/bin:${PATH} which clang++ - export PATH=$(./mason/mason prefix llvm-cov 3.9.1)/bin:${PATH} + export PATH=$(./mason/mason prefix llvm-cov 4.0.1)/bin:${PATH} which llvm-cov fi diff --git a/install_mason.sh b/install_mason.sh index 192b304b..1bd97c50 100755 --- a/install_mason.sh +++ b/install_mason.sh @@ -9,17 +9,18 @@ function install() { } ICU_VERSION="57.1" +MASON_VERSION="v1.14.2" if [ ! -f ./mason/mason.sh ]; then mkdir -p ./mason - curl -sSfL https://github.com/mapbox/mason/archive/9eac60614fda7cfeb8a9e81d18e8cca5c1ae8fbc.tar.gz | tar --gunzip --extract --strip-components=1 --exclude="*md" --exclude="test*" --directory=./mason + curl -sSfL https://github.com/mapbox/mason/archive/${MASON_VERSION}.tar.gz | tar --gunzip --extract --strip-components=1 --exclude="*md" --exclude="test*" --directory=./mason fi # core deps -install protozero 1.5.1 -install geometry 0.9.1 -install wagyu 0.4.2 -install protobuf 2.6.1 +install protozero 1.5.2 +install geometry 0.9.2 +install wagyu 0.4.3 +install protobuf 3.3.0 # mapnik if [[ ! ${SKIP_MAPNIK_INSTALL:-true} ]] && [[ ! -f ./mason_packages/.link/bin/mapnik-config ]]; then diff --git a/scripts/build.sh b/scripts/build.sh index 32481573..7021da09 100755 --- a/scripts/build.sh +++ b/scripts/build.sh @@ -2,10 +2,15 @@ set -e -u set -o pipefail if [[ ${COVERAGE:-false} == true ]]; then + # test only debug version export LDFLAGS="--coverage" export CXXFLAGS="--coverage" +else + # test both release and debug + make release -j${JOBS:-1} V=1 + make test + make debug -j${JOBS:-1} V=1 + make test-debug fi -make -j${JOBS:-1} test BUILDTYPE=${BUILDTYPE:-Release} V=1 - set +e +u From e6773b9ef6be86e8a9147f738d5cb3a5befc2193 Mon Sep 17 00:00:00 2001 From: Dane Springmeyer Date: Sat, 9 Sep 2017 17:30:45 -0700 Subject: [PATCH 4/9] fix mason version --- install_mason.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/install_mason.sh b/install_mason.sh index 1bd97c50..1ecd8e26 100755 --- a/install_mason.sh +++ b/install_mason.sh @@ -9,7 +9,7 @@ function install() { } ICU_VERSION="57.1" -MASON_VERSION="v1.14.2" +MASON_VERSION="v0.14.2" if [ ! -f ./mason/mason.sh ]; then mkdir -p ./mason From 4b4aa608a38b90aa4661c7d213ad67305e506a39 Mon Sep 17 00:00:00 2001 From: Dane Springmeyer Date: Sat, 9 Sep 2017 17:33:36 -0700 Subject: [PATCH 5/9] fix mapnik install logic via mason --- Makefile | 4 ++-- install_mason.sh | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index 580e064b..534125c8 100755 --- a/Makefile +++ b/Makefile @@ -3,10 +3,10 @@ GYP_REVISION=3464008 default: release mason_packages/.link/bin: - ./install_mason.sh + SKIP_MAPNIK_INSTALL=YES ./install_mason.sh mason_packages/.link/bin/mapnik-config: - SKIP_MAPNIK_INSTALL=YES ./install_mason.sh + ./install_mason.sh pre_build_check: @echo "Looking for mapnik-config on your PATH..." diff --git a/install_mason.sh b/install_mason.sh index 1ecd8e26..f20aea3c 100755 --- a/install_mason.sh +++ b/install_mason.sh @@ -23,7 +23,7 @@ install wagyu 0.4.3 install protobuf 3.3.0 # mapnik -if [[ ! ${SKIP_MAPNIK_INSTALL:-true} ]] && [[ ! -f ./mason_packages/.link/bin/mapnik-config ]]; then +if [[ ${SKIP_MAPNIK_INSTALL:-} != 'YES' ]] && [[ ! -f ./mason_packages/.link/bin/mapnik-config ]]; then # mapnik deps install jpeg_turbo 1.5.1 From eb2c3e8f19a14888fbae9e6d85461f7aa6ee0d84 Mon Sep 17 00:00:00 2001 From: Dane Springmeyer Date: Sat, 9 Sep 2017 18:25:18 -0700 Subject: [PATCH 6/9] fixup build logic --- Makefile | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/Makefile b/Makefile index 534125c8..96a0dd4f 100755 --- a/Makefile +++ b/Makefile @@ -17,18 +17,19 @@ pre_build_check: build/Makefile: ./deps/gyp gyp/build.gyp test/* deps/gyp/gyp gyp/build.gyp --depth=. -DMAPNIK_PLUGINDIR=\"$(shell mapnik-config --input-plugins)\" -Goutput_dir=. --generator-output=./build -f make + $(MAKE) -C build/ V=$(V) -release: mason_packages/.link/bin/mapnik-config build/Makefile Makefile - CXXFLAGS="-D_GLIBCXX_USE_CXX11_ABI=0" PATH="`pwd`/mason_packages/.link/bin/:${PATH}" $(MAKE) -C build/ BUILDTYPE=Release V=$(V) +release: mason_packages/.link/bin/mapnik-config Makefile + CXXFLAGS="-D_GLIBCXX_USE_CXX11_ABI=0" PATH="`pwd`/mason_packages/.link/bin/:${PATH}" $(MAKE) release_base -debug: mason_packages/.link/bin/mapnik-config build/Makefile Makefile - CXXFLAGS="-D_GLIBCXX_USE_CXX11_ABI=0" PATH="`pwd`/mason_packages/.link/bin/:${PATH}" $(MAKE) -C build/ BUILDTYPE=Debug V=$(V) +debug: mason_packages/.link/bin/mapnik-config Makefile + CXXFLAGS="-D_GLIBCXX_USE_CXX11_ABI=0" PATH="`pwd`/mason_packages/.link/bin/:${PATH}" $(MAKE) debug_base -release_base: pre_build_check mason_packages/.link/bin build/Makefile Makefile - $(MAKE) -C build/ BUILDTYPE=Release V=$(V) +release_base: pre_build_check mason_packages/.link/bin Makefile + BUILDTYPE=Release $(MAKE) build/Makefile -debug_base: pre_build_check mason_packages/.link/bin build/Makefile Makefile - $(MAKE) -C build/ BUILDTYPE=Debug V=$(V) +debug_base: pre_build_check mason_packages/.link/bin Makefile + BUILDTYPE=Debug $(MAKE) build/Makefile test/geometry-test-data/README.md: git submodule update --init @@ -49,6 +50,8 @@ clean: rm -rf ./build distclean: clean + rm -rf ./mason + rm -rf ./deps/gyp rm -rf ./mason_packages .PHONY: test From 8828c73de838835379544fb7de31107261ddcf68 Mon Sep 17 00:00:00 2001 From: Dane Springmeyer Date: Sat, 9 Sep 2017 18:29:22 -0700 Subject: [PATCH 7/9] more build fixes --- .travis.yml | 10 +++------- Makefile | 2 +- 2 files changed, 4 insertions(+), 8 deletions(-) diff --git a/.travis.yml b/.travis.yml index 1c442d2c..8fd7876f 100644 --- a/.travis.yml +++ b/.travis.yml @@ -17,7 +17,7 @@ matrix: - os: linux sudo: false compiler: ": gcc5" - env: JOBS=10 CXX="ccache g++-5" CXXFLAGS="-D_GLIBCXX_USE_CXX11_ABI=0" + env: JOBS=10 CXX="ccache g++-5" addons: apt: sources: ['ubuntu-toolchain-r-test'] @@ -25,7 +25,7 @@ matrix: - os: linux sudo: false compiler: ": clang" - env: JOBS=10 CXX="ccache clang++-3.9 -Qunused-arguments" + env: JOBS=10 CXX="ccache clang++-4.0 -Qunused-arguments" addons: apt: sources: ['ubuntu-toolchain-r-test'] @@ -33,7 +33,7 @@ matrix: - os: linux sudo: false compiler: ": clang-coverage" - env: JOBS=10 CXX="ccache clang++-3.9 -Qunused-arguments" COVERAGE=true + env: JOBS=10 CXX="ccache clang++-4.0 -Qunused-arguments" COVERAGE=true addons: apt: sources: ['ubuntu-toolchain-r-test'] @@ -65,7 +65,3 @@ before_script: script: # make sure tileinfo command works - ./build/${BUILDTYPE:-Release}/tileinfo examples/data/14_2620_6331.vector.mvt.z - -notifications: - slack: - secure: eP79cXpZ3zyVarMooLYi75C5XBS65SJkf12h25hLDmoaQD+aX0u3+/hivE3jxi8KI8v1Izpzf06aCGHL1lx/xXGg2uuiM6dtZTou1mJJ4oFsxzOJFfLXhsUO1+qXkJJG1xoaLpbSdX/y+2dfe0xKTm4zAlluj2DvwZO+R9djLGM= diff --git a/Makefile b/Makefile index 96a0dd4f..065eb434 100755 --- a/Makefile +++ b/Makefile @@ -54,6 +54,6 @@ distclean: clean rm -rf ./deps/gyp rm -rf ./mason_packages -.PHONY: test +.PHONY: test build/Makefile From 09a6366e205e224847b14fd362d35a5679d35176 Mon Sep 17 00:00:00 2001 From: Dane Springmeyer Date: Sat, 9 Sep 2017 18:37:19 -0700 Subject: [PATCH 8/9] fix coverage build --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 8fd7876f..42f2bcea 100644 --- a/.travis.yml +++ b/.travis.yml @@ -64,4 +64,4 @@ before_script: script: # make sure tileinfo command works - - ./build/${BUILDTYPE:-Release}/tileinfo examples/data/14_2620_6331.vector.mvt.z + - ./build/Debug/tileinfo examples/data/14_2620_6331.vector.mvt.z From 3d6ea56b242fb528b7d7c45874773ff57bad0b9f Mon Sep 17 00:00:00 2001 From: Dane Springmeyer Date: Sat, 9 Sep 2017 18:47:58 -0700 Subject: [PATCH 9/9] Actually fix coverage buildg --- scripts/build.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/scripts/build.sh b/scripts/build.sh index 7021da09..15640987 100755 --- a/scripts/build.sh +++ b/scripts/build.sh @@ -5,6 +5,8 @@ if [[ ${COVERAGE:-false} == true ]]; then # test only debug version export LDFLAGS="--coverage" export CXXFLAGS="--coverage" + make debug -j${JOBS:-1} V=1 + make test-debug else # test both release and debug make release -j${JOBS:-1} V=1