From 0e1504cdd74f5b8be9b59a460fc4e6105d54f3f9 Mon Sep 17 00:00:00 2001 From: muli Date: Thu, 8 Oct 2015 21:01:56 -0400 Subject: [PATCH 01/24] [travis] learn --- .travis.yml | 28 ++++++++++++++-------------- scripts/build_dmlc.sh | 27 --------------------------- scripts/travis_script.sh | 7 ++++++- 3 files changed, 20 insertions(+), 42 deletions(-) delete mode 100755 scripts/build_dmlc.sh diff --git a/.travis.yml b/.travis.yml index 88c837798c01..33e9467b2db9 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,5 +1,6 @@ # disable sudo to use container based build sudo: false + # Enabling test on Linux and OS X os: - linux @@ -7,16 +8,19 @@ os: # Use Build Matrix to do lint and build seperately env: + global: + - CXX=g++-4.8 matrix: - - TASK=lint LINT_LANG=cpp - - TASK=lint LINT_LANG=python + - TASK=lint - TASK=doc - - TASK=build CXX=g++ - - TASK=python CXX=g++ - - TASK=python3 CXX=g++ - - TASK=python_naive CXX=g++ - - TASK=python_perdev CXX=g++ - - TASK=cpp_unittest CXX=g++ + - TASK=build + - TASK=python_test + - TASK=cpp_test + +matrix: + exclude: + - os: osx + env: TASK=lint # dependent apt packages addons: @@ -33,7 +37,6 @@ addons: - libopencv-dev - gcc-4.8 - g++-4.8 - - clang - python-numpy - python-nose - python3-numpy @@ -43,21 +46,18 @@ addons: before_install: - export NVCC_PREFIX=${HOME} - - if [ "$CXX" = "g++" ]; then export CXX="g++-4.8" CC="gcc-4.8"; fi - - scripts/build_dmlc.sh - export TRAVIS=dmlc-core/scripts/travis - export PYTHONPATH=${PYTHONPATH}:${PWD}/python - source ${TRAVIS}/travis_setup_env.sh - + - git submodule init + - git submodule update install: - pip install cpplint pylint graphviz --user `whoami` - script: - scripts/travis_script.sh - before_cache: - ${TRAVIS}/travis_before_cache.sh diff --git a/scripts/build_dmlc.sh b/scripts/build_dmlc.sh deleted file mode 100755 index a5000499d084..000000000000 --- a/scripts/build_dmlc.sh +++ /dev/null @@ -1,27 +0,0 @@ -#! /bin/bash - -if [ ! -d mshadow ]; then - git clone https://github.com/dmlc/mshadow.git -fi - -if [ ! -d rabit ]; then - git clone https://github.com/dmlc/rabit.git -fi - -if [ ! -d dmlc-core ]; then - git clone https://github.com/dmlc/dmlc-core.git -fi - - -if [ ! -f config.mk ]; then - echo "Use the default config.m" - cp make/config.mk config.mk -fi - -cd rabit -make -j4 -cd .. - -cd dmlc-core -make -j4 -cd .. diff --git a/scripts/travis_script.sh b/scripts/travis_script.sh index a56b73c469b9..4dfe78489fe1 100755 --- a/scripts/travis_script.sh +++ b/scripts/travis_script.sh @@ -1,5 +1,11 @@ #!/bin/bash +echo ${TASK} +echo ${TRAVIS_OS_NAME} +echo ${CXX} + +exit 0 + # main script of travis if [ ${TASK} == "lint" ]; then if [ ${TRAVIS_OS_NAME} != "osx" ]; then @@ -93,4 +99,3 @@ if [ ${TASK} == "cpp_unittest" ]; then ./$test || exit -1 done fi - From 49bc380c6838c85f6cf94a3fe3f4613942408c5c Mon Sep 17 00:00:00 2001 From: muli Date: Thu, 8 Oct 2015 21:26:03 -0400 Subject: [PATCH 02/24] [travis] move travis to tests/travis --- .travis.yml | 12 +++++++----- scripts/travis_script.sh => tests/travis/run_test.sh | 2 ++ .../travis/setup_osx.sh | 0 3 files changed, 9 insertions(+), 5 deletions(-) rename scripts/travis_script.sh => tests/travis/run_test.sh (99%) rename scripts/travis_osx_install.sh => tests/travis/setup_osx.sh (100%) diff --git a/.travis.yml b/.travis.yml index 33e9467b2db9..b13e9548002d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,12 +1,14 @@ # disable sudo to use container based build sudo: false +language: cpp -# Enabling test on Linux and OS X +python: + - "2.7" + - "nightly" os: - linux - osx -# Use Build Matrix to do lint and build seperately env: global: - CXX=g++-4.8 @@ -21,6 +23,8 @@ matrix: exclude: - os: osx env: TASK=lint + - os: osx + env: TASK=doc # dependent apt packages addons: @@ -49,14 +53,12 @@ before_install: - export TRAVIS=dmlc-core/scripts/travis - export PYTHONPATH=${PYTHONPATH}:${PWD}/python - source ${TRAVIS}/travis_setup_env.sh - - git submodule init - - git submodule update install: - pip install cpplint pylint graphviz --user `whoami` script: - - scripts/travis_script.sh + - tests/travis/run_test.sh before_cache: - ${TRAVIS}/travis_before_cache.sh diff --git a/scripts/travis_script.sh b/tests/travis/run_test.sh similarity index 99% rename from scripts/travis_script.sh rename to tests/travis/run_test.sh index 4dfe78489fe1..9b6ad6469192 100755 --- a/scripts/travis_script.sh +++ b/tests/travis/run_test.sh @@ -1,5 +1,7 @@ #!/bin/bash +python --version + echo ${TASK} echo ${TRAVIS_OS_NAME} echo ${CXX} diff --git a/scripts/travis_osx_install.sh b/tests/travis/setup_osx.sh similarity index 100% rename from scripts/travis_osx_install.sh rename to tests/travis/setup_osx.sh From fbf02510b917d1c2adf7950755f3ed8ff77a8e7d Mon Sep 17 00:00:00 2001 From: muli Date: Thu, 8 Oct 2015 21:53:16 -0400 Subject: [PATCH 03/24] [travis] update --- .travis.yml | 26 ++++++++++++++------------ tests/travis/run_test.sh | 13 ++++++++----- 2 files changed, 22 insertions(+), 17 deletions(-) diff --git a/.travis.yml b/.travis.yml index b13e9548002d..9e08d8998b85 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,23 +1,25 @@ # disable sudo to use container based build sudo: false + language: cpp -python: - - "2.7" - - "nightly" os: - linux - osx env: - global: - - CXX=g++-4.8 - matrix: - - TASK=lint - - TASK=doc - - TASK=build - - TASK=python_test - - TASK=cpp_test + # code analysis + - TASK=lint + # test doc build + - TASK=doc + # build mxnet.so with CUDA + - TASK=build + # run tests/cpp + - TASK=cpp_test + # run tests/python + - TASK=python_test + + # TODO, R test, distributed test, clang, more g++ versions matrix: exclude: @@ -55,7 +57,7 @@ before_install: - source ${TRAVIS}/travis_setup_env.sh install: - - pip install cpplint pylint graphviz --user `whoami` + - echo 0 script: - tests/travis/run_test.sh diff --git a/tests/travis/run_test.sh b/tests/travis/run_test.sh index 9b6ad6469192..aecaa631609f 100755 --- a/tests/travis/run_test.sh +++ b/tests/travis/run_test.sh @@ -1,6 +1,11 @@ #!/bin/bash +if [ ${CXX} == "g++" ]; then + export CXX=g++-4.8 +fi + python --version +${CXX} --version echo ${TASK} echo ${TRAVIS_OS_NAME} @@ -8,12 +13,10 @@ echo ${CXX} exit 0 -# main script of travis if [ ${TASK} == "lint" ]; then - if [ ${TRAVIS_OS_NAME} != "osx" ]; then - make lint || exit -1 - fi - exit 0 + pip install cpplint pylint graphviz --user `whoami` + make lint + exit $? fi if [ ${TASK} == "doc" ]; then From f420b828c8b580dc18f73f72a64403fd05d134ed Mon Sep 17 00:00:00 2001 From: muli Date: Thu, 8 Oct 2015 22:06:04 -0400 Subject: [PATCH 04/24] [make] fix make doc warnings --- Makefile | 2 ++ include/mxnet/c_api.h | 14 +++++++------- include/mxnet/engine.h | 2 +- include/mxnet/kvstore.h | 4 ++-- include/mxnet/ndarray.h | 4 ++-- include/mxnet/resource.h | 2 +- src/common/lazy_alloc_array.h | 2 +- tests/travis/run_test.sh | 13 +++++-------- 8 files changed, 21 insertions(+), 22 deletions(-) diff --git a/Makefile b/Makefile index 49fbd151ef5f..2e9cf8f185cd 100644 --- a/Makefile +++ b/Makefile @@ -139,6 +139,8 @@ lint: rcpplint rcpplint: python dmlc-core/scripts/lint.py mxnet-rcpp ${LINT_LANG} R-package/src +doc: doxygen + doxygen: doxygen doc/Doxyfile diff --git a/include/mxnet/c_api.h b/include/mxnet/c_api.h index d3b21c057697..1dc302d0d77b 100644 --- a/include/mxnet/c_api.h +++ b/include/mxnet/c_api.h @@ -393,14 +393,14 @@ MXNET_DLL int MXSymbolCreateFromFile(const char *fname, SymbolHandle *out); MXNET_DLL int MXSymbolCreateFromJSON(const char *json, SymbolHandle *out); /*! * \brief Save a symbol into a json file. - * \param sym the input symbol. + * \param symbol the input symbol. * \param fname the file name. * \return 0 when success, -1 when failure happens */ MXNET_DLL int MXSymbolSaveToFile(SymbolHandle symbol, const char *fname); /*! * \brief Save a symbol into a json string - * \param sym the input symbol. + * \param symbol the input symbol. * \param out_json output json string. * \return 0 when success, -1 when failure happens */ @@ -551,7 +551,7 @@ MXNET_DLL int MXSymbolInferShape(SymbolHandle sym, * \param out_str pointer to hold the output string of the printing. * \return 0 when success, -1 when failure happens */ -MXNET_DLL int MXExecutorPrint(ExecutorHandle symbol, const char **out_str); +MXNET_DLL int MXExecutorPrint(ExecutorHandle handle, const char **out_str); /*! * \brief Executor forward method * @@ -759,7 +759,7 @@ MXNET_DLL int MXKVStorePull(KVStoreHandle handle, mx_uint num, int* keys, NDArrayHandle* vals, - int Priority); + int priority); /*! * \brief user-defined updater for the kvstore * It's this updater's responsibility to delete \a recv and \a local @@ -793,7 +793,7 @@ MXNET_DLL int MXKVStoreSetUpdater(KVStoreHandle handle, * \return 0 when success, -1 when failure happens */ MXNET_DLL int MXKVStoreGetRank(KVStoreHandle handle, - int *rank); + int *ret); /** * \brief return The number of nodes in this group, which is @@ -805,7 +805,7 @@ MXNET_DLL int MXKVStoreGetRank(KVStoreHandle handle, * \return 0 when success, -1 when failure happens */ MXNET_DLL int MXKVStoreGetGroupSize(KVStoreHandle handle, - int *size); + int *ret); /** * \brief return whether or not this process is a worker node. @@ -881,7 +881,7 @@ typedef void (MXKVStoreServerController)(int head, * \return Run as server (or scheduler) * * \param handle handle to the KVStore - * \param controler the user-defined server controller + * \param controller the user-defined server controller * \return 0 when success, -1 when failure happens */ MXNET_DLL int MXKVStoreRunServer(KVStoreHandle handle, diff --git a/include/mxnet/engine.h b/include/mxnet/engine.h index 065ddcb74aa6..03eb45b54de0 100644 --- a/include/mxnet/engine.h +++ b/include/mxnet/engine.h @@ -144,7 +144,7 @@ class Engine { * The delete will not happen immediately, but will wait until all the * operations depending on var are completed. * - * \param delete_fun A function that will be called after the variable is + * \param delete_fn A function that will be called after the variable is * deleted. * \param exec_ctx Execution context. * \param var The variable to be deleted. diff --git a/include/mxnet/kvstore.h b/include/mxnet/kvstore.h index 5126ebe3c5ba..da7d94a75cfd 100644 --- a/include/mxnet/kvstore.h +++ b/include/mxnet/kvstore.h @@ -78,7 +78,7 @@ class KVStore { * always has the same shape as being inited. * * \param keys the list of keys - * \param value the list of values + * \param values the list of values * \param priority Priority of the action. */ virtual void Push(const std::vector& keys, @@ -123,7 +123,7 @@ class KVStore { * value stored on the store node. The store updates \a y by `h(x, &y)`. The * default \a h is ASSIGN, namely `*y = x`. * - * \param updt user-defined updater, default is assign + * \param updater user-defined updater, default is assign */ void set_updater(Updater updater) { updater_ = updater; diff --git a/include/mxnet/ndarray.h b/include/mxnet/ndarray.h index f9ab9386e8c0..fce3d61f6855 100644 --- a/include/mxnet/ndarray.h +++ b/include/mxnet/ndarray.h @@ -246,7 +246,7 @@ class NDArray { * \brief Save list of narray into the file. * \param fname name of the file. * \param data the NDArrays to be saved. - * \param keys the name of the NDArray, optional, can be zero length. + * \param names the name of the NDArray, optional, can be zero length. */ static void Save(const std::string& fname, const std::vector& data, @@ -344,7 +344,7 @@ void CopyFromTo(const NDArray &from, NDArray *to, int priority = 0); /*! * \brief Perform elementwise sum over each data from source, store result into out. * \param source the ndarray we want to sum - * \param to the target ndarray + * \param out the target ndarray * \param priority Priority of the action. */ void ElementwiseSum(const std::vector &source, NDArray *out, int priority = 0); diff --git a/include/mxnet/resource.h b/include/mxnet/resource.h index 53a6dd6698e6..ded89adcb254 100644 --- a/include/mxnet/resource.h +++ b/include/mxnet/resource.h @@ -57,7 +57,7 @@ struct Resource { Resource() : id(0) {} /*! * \brief Get random number generator. - * \param The stream to use in the random number generator. + * \param stream The stream to use in the random number generator. * \return the mshadow random number generator requested. * \tparam xpu the device type of random number generator. */ diff --git a/src/common/lazy_alloc_array.h b/src/common/lazy_alloc_array.h index 28c4c259c603..d3722e557b04 100644 --- a/src/common/lazy_alloc_array.h +++ b/src/common/lazy_alloc_array.h @@ -29,7 +29,7 @@ class LazyAllocArray { inline TElem* Get(int index, FCreate creator); /*! * \brief for each not null element of the array, call fvisit - * \param fviist a function of (size_t, TElem*) + * \param fvisit a function of (size_t, TElem*) */ template inline void ForEach(FVisit fvisit); diff --git a/tests/travis/run_test.sh b/tests/travis/run_test.sh index aecaa631609f..92efdec60ff6 100755 --- a/tests/travis/run_test.sh +++ b/tests/travis/run_test.sh @@ -11,22 +11,19 @@ echo ${TASK} echo ${TRAVIS_OS_NAME} echo ${CXX} -exit 0 - if [ ${TASK} == "lint" ]; then - pip install cpplint pylint graphviz --user `whoami` + pip install cpplint pylint --user `whoami` make lint exit $? fi if [ ${TASK} == "doc" ]; then - if [ ${TRAVIS_OS_NAME} != "osx" ]; then - make doc 2>log.txt - (cat log.txt|grep warning) && exit -1 - fi - exit 0 + make doc 2>log.txt + if [ $? -ne 0 ]; exit $?; fi + (cat log.txt|grep warning) && exit -1 fi +exit 0 # prereqs for things that need make cp make/config.mk config.mk From b1582d404cc9b23f2c94bd0bd40282acf3156500 Mon Sep 17 00:00:00 2001 From: muli Date: Thu, 8 Oct 2015 22:14:16 -0400 Subject: [PATCH 05/24] [travis] more --- .travis.yml | 1 - tests/travis/run_test.sh | 27 +++++++++++++++------------ 2 files changed, 15 insertions(+), 13 deletions(-) diff --git a/.travis.yml b/.travis.yml index 9e08d8998b85..26dc71520a54 100644 --- a/.travis.yml +++ b/.travis.yml @@ -51,7 +51,6 @@ addons: before_install: - - export NVCC_PREFIX=${HOME} - export TRAVIS=dmlc-core/scripts/travis - export PYTHONPATH=${PYTHONPATH}:${PWD}/python - source ${TRAVIS}/travis_setup_env.sh diff --git a/tests/travis/run_test.sh b/tests/travis/run_test.sh index 92efdec60ff6..742eb6ba0092 100755 --- a/tests/travis/run_test.sh +++ b/tests/travis/run_test.sh @@ -19,14 +19,27 @@ fi if [ ${TASK} == "doc" ]; then make doc 2>log.txt - if [ $? -ne 0 ]; exit $?; fi (cat log.txt|grep warning) && exit -1 + exit 0 fi -exit 0 +# more setups + # prereqs for things that need make cp make/config.mk config.mk +if [ ${TASK} == "build" ]; then + if [ ${TRAVIS_OS_NAME} == "linux" ]; then + echo "USE_CUDA=1" >> config.mk + export NVCC_PREFIX=${HOME} + ./dmlc-core/scripts/setup_nvcc.sh $NVCC_PREFIX + fi + make all + exit $? +fi + +exit 0 + export NOSE3=nosetests3 export PYTHON3=python3 if [ ${TRAVIS_OS_NAME} == "osx" ]; then @@ -43,16 +56,6 @@ else export CXX="g++-4.8" fi -echo "USE_S3=0" >> config.mk - -if [ ${TASK} == "build" ]; then - if [ ${TRAVIS_OS_NAME} != "osx" ]; then - echo "USE_CUDA=1" >> config.mk - echo "USE_THREADED_ENGINE=1" >> config.mk - ./dmlc-core/scripts/setup_nvcc.sh $NVCC_PREFIX - make all || exit -1 - fi -fi if [ ${TASK} == "python" ]; then echo "USE_CUDA=0" >> config.mk From eb986675f0d4e970d4f67d14fe8a69b12759db82 Mon Sep 17 00:00:00 2001 From: muli Date: Thu, 8 Oct 2015 22:39:35 -0400 Subject: [PATCH 06/24] [travis] update travis --- tests/travis/run_test.sh | 32 ++++++++++++++++++++++++-------- 1 file changed, 24 insertions(+), 8 deletions(-) diff --git a/tests/travis/run_test.sh b/tests/travis/run_test.sh index 742eb6ba0092..5999271b27f5 100755 --- a/tests/travis/run_test.sh +++ b/tests/travis/run_test.sh @@ -1,9 +1,5 @@ #!/bin/bash -if [ ${CXX} == "g++" ]; then - export CXX=g++-4.8 -fi - python --version ${CXX} --version @@ -18,7 +14,7 @@ if [ ${TASK} == "lint" ]; then fi if [ ${TASK} == "doc" ]; then - make doc 2>log.txt + make doc | tee 2>log.txt (cat log.txt|grep warning) && exit -1 exit 0 fi @@ -28,6 +24,18 @@ fi # prereqs for things that need make cp make/config.mk config.mk +if [ ${CXX} == "g++" ]; then + export CXX=g++-4.8 +fi +echo "CXX=${CXX}" >>config.mk + +if [ ${TRAVIS_OS_NAME} == "osx" ]; then + echo "USE_BLAS=apple" >> config.mk + echo "USE_OPENMP=0" >> config.mk +else + echo "USE_BLAS=blas" >> config.mk +fi + if [ ${TASK} == "build" ]; then if [ ${TRAVIS_OS_NAME} == "linux" ]; then echo "USE_CUDA=1" >> config.mk @@ -38,19 +46,27 @@ if [ ${TASK} == "build" ]; then exit $? fi +if [ ${TASK} == "cpp_test" ]; then + make -f dmlc-core/scripts/packages.mk gtest + echo "GTEST_PATH="${CACHE_PREFIX} >> config.mk + make test || exit -1 + export MXNET_ENGINE_INFO=true + for test in tests/cpp/*_test; do + ./$test || exit -1 + done + exit 0 +fi + exit 0 export NOSE3=nosetests3 export PYTHON3=python3 if [ ${TRAVIS_OS_NAME} == "osx" ]; then source scripts/travis_osx_install.sh - echo "USE_BLAS=apple" >> config.mk - echo "USE_OPENMP=0" >> config.mk alias nosetests='python -m nose' export NOSE3='python -m nose' export PYTHON3=python else - echo "USE_BLAS=blas" >> config.mk echo "USE_CUDNN=0" >> config.mk echo "CXX=g++-4.8" >> config.mk export CXX="g++-4.8" From 455dbe01b0230f0c116c651869276a7d0f8c3a9a Mon Sep 17 00:00:00 2001 From: muli Date: Thu, 8 Oct 2015 23:02:44 -0400 Subject: [PATCH 07/24] [travis] update --- .travis.yml | 13 +++++-------- tests/travis/run_test.sh | 3 +-- 2 files changed, 6 insertions(+), 10 deletions(-) diff --git a/.travis.yml b/.travis.yml index 26dc71520a54..fd9f647b1300 100644 --- a/.travis.yml +++ b/.travis.yml @@ -51,19 +51,16 @@ addons: before_install: - - export TRAVIS=dmlc-core/scripts/travis - - export PYTHONPATH=${PYTHONPATH}:${PWD}/python - - source ${TRAVIS}/travis_setup_env.sh + - export NVCC_PREFIX=${HOME} + - source dmlc-core/scripts/travis/travis_setup_env.sh install: - - echo 0 + - if [ ${TRAVIS_OS_NAME} == "osx" ]; then tests/travis/setup_osx.sh; fi + - if [ ${TASK} == "lint" ]; then pip install cpplint pylint --user `whoami`; fi script: - tests/travis/run_test.sh -before_cache: - - ${TRAVIS}/travis_before_cache.sh - cache: directories: - ${HOME}/.cache/usr @@ -74,4 +71,4 @@ notifications: email: on_success: change on_failure: always - slack: dmlc:NmroCzntCiWOuxUZpii40USd +# slack: dmlc:NmroCzntCiWOuxUZpii40USd diff --git a/tests/travis/run_test.sh b/tests/travis/run_test.sh index 5999271b27f5..b576a9b83b64 100755 --- a/tests/travis/run_test.sh +++ b/tests/travis/run_test.sh @@ -1,5 +1,6 @@ #!/bin/bash + # - export PYTHONPATH=${PYTHONPATH}:${PWD}/python python --version ${CXX} --version @@ -8,7 +9,6 @@ echo ${TRAVIS_OS_NAME} echo ${CXX} if [ ${TASK} == "lint" ]; then - pip install cpplint pylint --user `whoami` make lint exit $? fi @@ -39,7 +39,6 @@ fi if [ ${TASK} == "build" ]; then if [ ${TRAVIS_OS_NAME} == "linux" ]; then echo "USE_CUDA=1" >> config.mk - export NVCC_PREFIX=${HOME} ./dmlc-core/scripts/setup_nvcc.sh $NVCC_PREFIX fi make all From 7b35de26d12447bc91db6ca60b403b293c974f08 Mon Sep 17 00:00:00 2001 From: muli Date: Thu, 8 Oct 2015 23:14:38 -0400 Subject: [PATCH 08/24] [travis] update --- tests/travis/run_test.sh | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/tests/travis/run_test.sh b/tests/travis/run_test.sh index b576a9b83b64..154b6623905e 100755 --- a/tests/travis/run_test.sh +++ b/tests/travis/run_test.sh @@ -56,6 +56,25 @@ if [ ${TASK} == "cpp_test" ]; then exit 0 fi +if [ ${TASK} == "python_test" ]; then + make all || exit -1 + rm -rf tests/python/data + mkdir -p ${CACHE_PREFIX}/data + ln -s ${CACHE_PREFIX}/data tests/python/data + python --version + if [ ${TRAVIS_OS_NAME} == "osx" ]; then + alias nosetests='python -m nose' + fi + nosetests tests/python/unittest || exit -1 + nosetests tests/python/train || exit -1 + + if [ ${TRAVIS_OS_NAME} == "linux" ]; then + python3 --version + nosetests3 tests/python/unittest || exit -1 + nosetests3 tests/python/train || exit -1 + fi +fi + exit 0 export NOSE3=nosetests3 From dad98a2fd863323dfab5cf167dbccc1d650222c7 Mon Sep 17 00:00:00 2001 From: muli Date: Thu, 8 Oct 2015 23:19:12 -0400 Subject: [PATCH 09/24] [travis] hope to fix mac os x --- tests/travis/run_test.sh | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/tests/travis/run_test.sh b/tests/travis/run_test.sh index 154b6623905e..05229ebd8398 100755 --- a/tests/travis/run_test.sh +++ b/tests/travis/run_test.sh @@ -24,17 +24,18 @@ fi # prereqs for things that need make cp make/config.mk config.mk -if [ ${CXX} == "g++" ]; then - export CXX=g++-4.8 -fi -echo "CXX=${CXX}" >>config.mk if [ ${TRAVIS_OS_NAME} == "osx" ]; then echo "USE_BLAS=apple" >> config.mk echo "USE_OPENMP=0" >> config.mk else + # use g++-4.8 for linux + if [ ${CXX} == "g++" ]; then + export CXX=g++-4.8 + fi echo "USE_BLAS=blas" >> config.mk fi +echo "CXX=${CXX}" >>config.mk if [ ${TASK} == "build" ]; then if [ ${TRAVIS_OS_NAME} == "linux" ]; then From 122f6545ceb34db599d537743f39e041cff1ce41 Mon Sep 17 00:00:00 2001 From: muli Date: Thu, 8 Oct 2015 23:27:03 -0400 Subject: [PATCH 10/24] [travis] update --- tests/travis/run_test.sh | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tests/travis/run_test.sh b/tests/travis/run_test.sh index 05229ebd8398..051a2ec31070 100755 --- a/tests/travis/run_test.sh +++ b/tests/travis/run_test.sh @@ -59,18 +59,23 @@ fi if [ ${TASK} == "python_test" ]; then make all || exit -1 + # use cached dir for storing data rm -rf tests/python/data mkdir -p ${CACHE_PREFIX}/data ln -s ${CACHE_PREFIX}/data tests/python/data + python --version if [ ${TRAVIS_OS_NAME} == "osx" ]; then alias nosetests='python -m nose' fi + python python/setup.py develop --user nosetests tests/python/unittest || exit -1 nosetests tests/python/train || exit -1 if [ ${TRAVIS_OS_NAME} == "linux" ]; then python3 --version + rm -rf python/mxnet/mxnet.egg-info + python3 python/setup.py develop --user nosetests3 tests/python/unittest || exit -1 nosetests3 tests/python/train || exit -1 fi From eb768c6d8f8f98aae9a7512a3c5103edffc24071 Mon Sep 17 00:00:00 2001 From: muli Date: Thu, 8 Oct 2015 23:37:04 -0400 Subject: [PATCH 11/24] [make] delete lib and bin, try to fix travis python --- .travis.yml | 2 +- Makefile | 5 ++++- bin/README | 1 - lib/README.md | 1 - tests/travis/run_test.sh | 6 +++--- 5 files changed, 8 insertions(+), 7 deletions(-) delete mode 100644 bin/README delete mode 100644 lib/README.md diff --git a/.travis.yml b/.travis.yml index fd9f647b1300..6ce1795a3b67 100644 --- a/.travis.yml +++ b/.travis.yml @@ -56,7 +56,7 @@ before_install: install: - if [ ${TRAVIS_OS_NAME} == "osx" ]; then tests/travis/setup_osx.sh; fi - - if [ ${TASK} == "lint" ]; then pip install cpplint pylint --user `whoami`; fi + - if [ ${TASK} == "lint" ]; then pip install cpplint pylint graphviz --user `whoami`; fi script: - tests/travis/run_test.sh diff --git a/Makefile b/Makefile index 2e9cf8f185cd..638568f68ad7 100644 --- a/Makefile +++ b/Makefile @@ -111,9 +111,11 @@ build/%_gpu.o: src/%.cu $(LIB_DEP) $(NVCC) -c -o $@ $(NVCCFLAGS) -Xcompiler "$(CFLAGS)" $< lib/libmxnet.a: $(ALL_DEP) + @mkdir -p $(@D) ar crv $@ $(filter %.o, $?) lib/libmxnet.so: $(ALL_DEP) + @mkdir -p $(@D) $(CXX) $(CFLAGS) -shared -o $@ $(filter %.o %.a, $^) $(LDFLAGS) # ps-lite @@ -127,6 +129,7 @@ $(DMLC_CORE)/libdmlc.a: bin/im2rec: tools/im2rec.cc $(DMLC_CORE)/libdmlc.a $(BIN) : + @mkdir -p $(@D) $(CXX) $(CFLAGS) -o $@ $(filter %.cpp %.o %.c %.a %.cc, $^) $(LDFLAGS) include tests/cpp/unittest.mk @@ -145,7 +148,7 @@ doxygen: doxygen doc/Doxyfile clean: - $(RM) -r build lib/lib* *~ */*~ */*/*~ */*/*/*~ + $(RM) -r build lib bin *~ */*~ */*/*~ */*/*/*~ clean_all: clean cd $(DMLC_CORE); make clean; cd - diff --git a/bin/README b/bin/README deleted file mode 100644 index c44ad0da7d67..000000000000 --- a/bin/README +++ /dev/null @@ -1 +0,0 @@ -This is where the executable files of mxnet tools will be generated into. diff --git a/lib/README.md b/lib/README.md deleted file mode 100644 index 24d68ff1acba..000000000000 --- a/lib/README.md +++ /dev/null @@ -1 +0,0 @@ -MXNet library diff --git a/tests/travis/run_test.sh b/tests/travis/run_test.sh index 051a2ec31070..c752d07d2883 100755 --- a/tests/travis/run_test.sh +++ b/tests/travis/run_test.sh @@ -68,14 +68,14 @@ if [ ${TASK} == "python_test" ]; then if [ ${TRAVIS_OS_NAME} == "osx" ]; then alias nosetests='python -m nose' fi - python python/setup.py develop --user + cd python && python setup.py develop --user && cd .. || exit -1 nosetests tests/python/unittest || exit -1 nosetests tests/python/train || exit -1 if [ ${TRAVIS_OS_NAME} == "linux" ]; then python3 --version - rm -rf python/mxnet/mxnet.egg-info - python3 python/setup.py develop --user + rm -rf python/mxnet.egg-info + cd python && python3 setup.py develop --user && cd .. || exit -1 nosetests3 tests/python/unittest || exit -1 nosetests3 tests/python/train || exit -1 fi From 663898193c2ef9dfc6893f0e17f8b14ea1e48f6d Mon Sep 17 00:00:00 2001 From: muli Date: Thu, 8 Oct 2015 23:49:27 -0400 Subject: [PATCH 12/24] [travis] fix python --- tests/travis/run_test.sh | 21 ++++++++------------- tests/travis/setup_osx.sh | 3 ++- 2 files changed, 10 insertions(+), 14 deletions(-) diff --git a/tests/travis/run_test.sh b/tests/travis/run_test.sh index c752d07d2883..042214925bb8 100755 --- a/tests/travis/run_test.sh +++ b/tests/travis/run_test.sh @@ -19,12 +19,8 @@ if [ ${TASK} == "doc" ]; then exit 0 fi -# more setups - -# prereqs for things that need make cp make/config.mk config.mk - if [ ${TRAVIS_OS_NAME} == "osx" ]; then echo "USE_BLAS=apple" >> config.mk echo "USE_OPENMP=0" >> config.mk @@ -72,13 +68,14 @@ if [ ${TASK} == "python_test" ]; then nosetests tests/python/unittest || exit -1 nosetests tests/python/train || exit -1 - if [ ${TRAVIS_OS_NAME} == "linux" ]; then - python3 --version - rm -rf python/mxnet.egg-info - cd python && python3 setup.py develop --user && cd .. || exit -1 - nosetests3 tests/python/unittest || exit -1 - nosetests3 tests/python/train || exit -1 - fi + # TODO(mli) python3 error + # if [ ${TRAVIS_OS_NAME} == "linux" ]; then + # python3 --version + # rm -rf python/mxnet.egg-info + # cd python && python3 setup.py develop --user && cd .. || exit -1 + # nosetests3 tests/python/unittest || exit -1 + # nosetests3 tests/python/train || exit -1 + # fi fi exit 0 @@ -94,8 +91,6 @@ else echo "USE_CUDNN=0" >> config.mk echo "CXX=g++-4.8" >> config.mk export CXX="g++-4.8" -fi - if [ ${TASK} == "python" ]; then echo "USE_CUDA=0" >> config.mk diff --git a/tests/travis/setup_osx.sh b/tests/travis/setup_osx.sh index 04929633ee5a..66b97cc10148 100755 --- a/tests/travis/setup_osx.sh +++ b/tests/travis/setup_osx.sh @@ -6,6 +6,7 @@ brew info opencv brew install graphviz brew install opencv +# TODO(mli) TASK are not correct... if [ ${TASK} == "python-package3" ]; then wget -O conda.sh https://repo.continuum.io/miniconda/Miniconda3-latest-MacOSX-x86_64.sh else @@ -28,4 +29,4 @@ else fi source activate myenv conda install numpy scipy matplotlib nose -python -m pip install graphviz \ No newline at end of file +python -m pip install graphviz From 67376a7d840b7572e2b226ca6c5752c7496e1d17 Mon Sep 17 00:00:00 2001 From: muli Date: Fri, 9 Oct 2015 00:03:37 -0400 Subject: [PATCH 13/24] [travis] fix mac os x python --- .travis.yml | 2 + tests/travis/run_test.sh | 89 +++++++--------------------------------- 2 files changed, 16 insertions(+), 75 deletions(-) diff --git a/.travis.yml b/.travis.yml index 6ce1795a3b67..38d8d050154b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -65,6 +65,8 @@ cache: directories: - ${HOME}/.cache/usr +before_cache: + - dmlc-core/scripts/travis/travis_before_cache.sh notifications: # Emails are sent to the committer's git-configured email address by default, diff --git a/tests/travis/run_test.sh b/tests/travis/run_test.sh index 042214925bb8..9045ff4b701f 100755 --- a/tests/travis/run_test.sh +++ b/tests/travis/run_test.sh @@ -61,81 +61,20 @@ if [ ${TASK} == "python_test" ]; then ln -s ${CACHE_PREFIX}/data tests/python/data python --version - if [ ${TRAVIS_OS_NAME} == "osx" ]; then - alias nosetests='python -m nose' - fi cd python && python setup.py develop --user && cd .. || exit -1 - nosetests tests/python/unittest || exit -1 - nosetests tests/python/train || exit -1 - - # TODO(mli) python3 error - # if [ ${TRAVIS_OS_NAME} == "linux" ]; then - # python3 --version - # rm -rf python/mxnet.egg-info - # cd python && python3 setup.py develop --user && cd .. || exit -1 - # nosetests3 tests/python/unittest || exit -1 - # nosetests3 tests/python/train || exit -1 - # fi -fi - -exit 0 - -export NOSE3=nosetests3 -export PYTHON3=python3 -if [ ${TRAVIS_OS_NAME} == "osx" ]; then - source scripts/travis_osx_install.sh - alias nosetests='python -m nose' - export NOSE3='python -m nose' - export PYTHON3=python -else - echo "USE_CUDNN=0" >> config.mk - echo "CXX=g++-4.8" >> config.mk - export CXX="g++-4.8" - -if [ ${TASK} == "python" ]; then - echo "USE_CUDA=0" >> config.mk - make all || exit -1 - python --version - export MXNET_ENGINE_TYPE=ThreadedEngine - nosetests tests/python/unittest || exit -1 - nosetests tests/python/train || exit -1 -fi - -if [ ${TASK} == "python3" ]; then - echo "USE_CUDA=0" >> config.mk - make all || exit -1 - export MXNET_ENGINE_TYPE=ThreadedEngine - ${PYTHON3} --version - ${NOSE3} tests/python/unittest || exit -1 - ${NOSE3} tests/python/train || exit -1 -fi - -if [ ${TASK} == "python_naive" ]; then - echo "USE_CUDA=0" >> config.mk - make all || exit -1 - export MXNET_ENGINE_TYPE=NaiveEngine - python --version - nosetests tests/python/unittest || exit -1 - nosetests tests/python/train || exit -1 -fi - -if [ ${TASK} == "python_perdev" ]; then - echo "USE_CUDA=0" >> config.mk - make all || exit -1 - export MXNET_ENGINE_TYPE=ThreadedEnginePerDevice - python --version - nosetests tests/python/unittest || exit -1 - nosetests tests/python/train || exit -1 -fi -if [ ${TASK} == "cpp_unittest" ]; then - make -f dmlc-core/scripts/packages.mk gtest - echo "USE_CUDA=0" >> config.mk - echo "GTEST_PATH="${CACHE_PREFIX} >> config.mk - make test || exit -1 - export MXNET_ENGINE_TYPE=ThreadedEngine - export MXNET_ENGINE_INFO=true - for test in tests/cpp/*_test; do - ./$test || exit -1 - done + if [ ${TRAVIS_OS_NAME} == "osx" ]; then + python -m nose tests/python/unittest || exit -1 + python -m nose tests/python/train || exit -1 + else + nosetests tests/python/unittest || exit -1 + nosetests tests/python/train || exit -1 + + # TODO(mli) python3 error + # python3 --version + # rm -rf python/mxnet.egg-info + # cd python && python3 setup.py develop --user && cd .. || exit -1 + # nosetests3 tests/python/unittest || exit -1 + # nosetests3 tests/python/train || exit -1 + fi fi From eef0401c8cec4acd9789e25da808f6ba43aeef78 Mon Sep 17 00:00:00 2001 From: muli Date: Fri, 9 Oct 2015 00:22:40 -0400 Subject: [PATCH 14/24] [travis] mac os x again --- .travis.yml | 19 +++++++++-------- tests/travis/setup_osx.sh | 43 ++++++++++++++++++++------------------- 2 files changed, 32 insertions(+), 30 deletions(-) diff --git a/.travis.yml b/.travis.yml index 38d8d050154b..9258c21b17a8 100644 --- a/.travis.yml +++ b/.travis.yml @@ -8,15 +8,15 @@ os: - osx env: - # code analysis - - TASK=lint - # test doc build - - TASK=doc - # build mxnet.so with CUDA - - TASK=build - # run tests/cpp - - TASK=cpp_test - # run tests/python + # # code analysis + # - TASK=lint + # # test doc build + # - TASK=doc + # # build mxnet.so with CUDA + # - TASK=build + # # run tests/cpp + # - TASK=cpp_test + # # run tests/python - TASK=python_test # TODO, R test, distributed test, clang, more g++ versions @@ -53,6 +53,7 @@ addons: before_install: - export NVCC_PREFIX=${HOME} - source dmlc-core/scripts/travis/travis_setup_env.sh + - export PYTHONPATH=${PYTHONPATH}:${PWD}/python install: - if [ ${TRAVIS_OS_NAME} == "osx" ]; then tests/travis/setup_osx.sh; fi diff --git a/tests/travis/setup_osx.sh b/tests/travis/setup_osx.sh index 66b97cc10148..876050970506 100755 --- a/tests/travis/setup_osx.sh +++ b/tests/travis/setup_osx.sh @@ -3,30 +3,31 @@ brew update brew tap homebrew/science brew info opencv -brew install graphviz +# brew install graphviz brew install opencv # TODO(mli) TASK are not correct... -if [ ${TASK} == "python-package3" ]; then - wget -O conda.sh https://repo.continuum.io/miniconda/Miniconda3-latest-MacOSX-x86_64.sh -else - wget -O conda.sh https://repo.continuum.io/miniconda/Miniconda-latest-MacOSX-x86_64.sh -fi +# if [ ${TASK} == "python-package3" ]; then +# wget -O conda.sh https://repo.continuum.io/miniconda/Miniconda3-latest-MacOSX-x86_64.sh +# else +# wget -O conda.sh https://repo.continuum.io/miniconda/Miniconda-latest-MacOSX-x86_64.sh +# fi -bash conda.sh -b -p $HOME/miniconda -export PATH="$HOME/miniconda/bin:$PATH" -hash -r -conda config --set always_yes yes --set changeps1 no -conda update -q conda -# Useful for debugging any issues with conda -conda info -a +# bash conda.sh -b -p $HOME/miniconda +# export PATH="$HOME/miniconda/bin:$PATH" +# hash -r +# conda config --set always_yes yes --set changeps1 no +# conda update -q conda +# # Useful for debugging any issues with conda +# conda info -a -if [ ${TASK} == "package3" ]; then - conda create -n myenv python=3.4 -else - conda create -n myenv python=2.7 -fi -source activate myenv -conda install numpy scipy matplotlib nose -python -m pip install graphviz +# if [ ${TASK} == "package3" ]; then +# conda create -n myenv python=3.4 +# else +# conda create -n myenv python=2.7 +# fi +# source activate myenv +# conda install numpy scipy matplotlib nose + +python -m pip install nose numpy From c477ba9109ce54d32962c0e321b86d2bdf14e268 Mon Sep 17 00:00:00 2001 From: muli Date: Fri, 9 Oct 2015 00:26:48 -0400 Subject: [PATCH 15/24] [travis] update osx --- tests/travis/setup_osx.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/travis/setup_osx.sh b/tests/travis/setup_osx.sh index 876050970506..596abb24a4db 100755 --- a/tests/travis/setup_osx.sh +++ b/tests/travis/setup_osx.sh @@ -30,4 +30,4 @@ brew install opencv # source activate myenv # conda install numpy scipy matplotlib nose -python -m pip install nose numpy +python -m pip install nose numpy --user `whoami` From 56d4d5de73bc43ae7ca7519afebb286554646723 Mon Sep 17 00:00:00 2001 From: muli Date: Fri, 9 Oct 2015 00:38:41 -0400 Subject: [PATCH 16/24] [travis] python --- .travis.yml | 3 +-- tests/travis/run_test.sh | 35 ++++++++++++++++++++--------------- tests/travis/setup.sh | 18 ++++++++++++++++++ tests/travis/setup_osx.sh | 3 ++- 4 files changed, 41 insertions(+), 18 deletions(-) create mode 100755 tests/travis/setup.sh diff --git a/.travis.yml b/.travis.yml index 9258c21b17a8..3c2b841888a0 100644 --- a/.travis.yml +++ b/.travis.yml @@ -56,8 +56,7 @@ before_install: - export PYTHONPATH=${PYTHONPATH}:${PWD}/python install: - - if [ ${TRAVIS_OS_NAME} == "osx" ]; then tests/travis/setup_osx.sh; fi - - if [ ${TASK} == "lint" ]; then pip install cpplint pylint graphviz --user `whoami`; fi + - source tests/travis/setup.sh script: - tests/travis/run_test.sh diff --git a/tests/travis/run_test.sh b/tests/travis/run_test.sh index 9045ff4b701f..0b9a649a2f1d 100755 --- a/tests/travis/run_test.sh +++ b/tests/travis/run_test.sh @@ -60,21 +60,26 @@ if [ ${TASK} == "python_test" ]; then mkdir -p ${CACHE_PREFIX}/data ln -s ${CACHE_PREFIX}/data tests/python/data - python --version - cd python && python setup.py develop --user && cd .. || exit -1 + # python --version + # cd python && python setup.py develop --user && cd .. || exit -1 - if [ ${TRAVIS_OS_NAME} == "osx" ]; then - python -m nose tests/python/unittest || exit -1 - python -m nose tests/python/train || exit -1 - else - nosetests tests/python/unittest || exit -1 - nosetests tests/python/train || exit -1 + python -m nose tests/python/unittest || exit -1 + python -m nose tests/python/train || exit -1 - # TODO(mli) python3 error - # python3 --version - # rm -rf python/mxnet.egg-info - # cd python && python3 setup.py develop --user && cd .. || exit -1 - # nosetests3 tests/python/unittest || exit -1 - # nosetests3 tests/python/train || exit -1 - fi + python3 -m nose tests/python/unittest || exit -1 + python3 -m nose tests/python/train || exit -1 + + # if [ ${TRAVIS_OS_NAME} == "osx" ]; then + # else + + # nosetests tests/python/unittest || exit -1 + # nosetests tests/python/train || exit -1 + + # # TODO(mli) python3 error + # # python3 --version + # # rm -rf python/mxnet.egg-info + # # cd python && python3 setup.py develop --user && cd .. || exit -1 + # # nosetests3 tests/python/unittest || exit -1 + # # nosetests3 tests/python/train || exit -1 + # fi fi diff --git a/tests/travis/setup.sh b/tests/travis/setup.sh new file mode 100755 index 000000000000..b059c1ec1a97 --- /dev/null +++ b/tests/travis/setup.sh @@ -0,0 +1,18 @@ +#!/bin/bash + +if [ ${TRAVIS_OS_NAME} == "osx" ]; then + brew update + brew tap homebrew/science + brew info opencv + brew install opencv + brew install python3 +fi + +if [ ${TASK} == "lint" ]; then + pip install cpplint pylint graphviz --user `whoami` +fi + +if [ ${TASK} == "python_test"]; then + python -m pip install nose numpy --user `whoami` + python3 -m pip install nose numpy --user `whoami` +fi diff --git a/tests/travis/setup_osx.sh b/tests/travis/setup_osx.sh index 596abb24a4db..a35e67e761ca 100755 --- a/tests/travis/setup_osx.sh +++ b/tests/travis/setup_osx.sh @@ -5,6 +5,7 @@ brew tap homebrew/science brew info opencv # brew install graphviz brew install opencv +brew install python3 # TODO(mli) TASK are not correct... # if [ ${TASK} == "python-package3" ]; then @@ -30,4 +31,4 @@ brew install opencv # source activate myenv # conda install numpy scipy matplotlib nose -python -m pip install nose numpy --user `whoami` +# python -m pip install nose numpy --user `whoami` From 09b9520124c1d280a8e5bb77cbe055ae56287e6a Mon Sep 17 00:00:00 2001 From: muli Date: Fri, 9 Oct 2015 00:46:41 -0400 Subject: [PATCH 17/24] [travis] fix --- tests/travis/setup.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/travis/setup.sh b/tests/travis/setup.sh index b059c1ec1a97..ef4ee3810552 100755 --- a/tests/travis/setup.sh +++ b/tests/travis/setup.sh @@ -12,7 +12,7 @@ if [ ${TASK} == "lint" ]; then pip install cpplint pylint graphviz --user `whoami` fi -if [ ${TASK} == "python_test"]; then +if [ ${TASK} == "python_test" ]; then python -m pip install nose numpy --user `whoami` python3 -m pip install nose numpy --user `whoami` fi From 8060dd5a367e9dd241b70441b0147d2b349a8b63 Mon Sep 17 00:00:00 2001 From: muli Date: Fri, 9 Oct 2015 00:59:37 -0400 Subject: [PATCH 18/24] [travis] python --- tests/travis/run_test.sh | 20 +++++++++++--------- tests/travis/setup.sh | 9 ++++----- 2 files changed, 15 insertions(+), 14 deletions(-) diff --git a/tests/travis/run_test.sh b/tests/travis/run_test.sh index 0b9a649a2f1d..225c6fdfe18a 100755 --- a/tests/travis/run_test.sh +++ b/tests/travis/run_test.sh @@ -63,23 +63,25 @@ if [ ${TASK} == "python_test" ]; then # python --version # cd python && python setup.py develop --user && cd .. || exit -1 - python -m nose tests/python/unittest || exit -1 - python -m nose tests/python/train || exit -1 - - python3 -m nose tests/python/unittest || exit -1 - python3 -m nose tests/python/train || exit -1 + if [ ${TRAVIS_OS_NAME} == "osx" ]; then + python -m nose tests/python/unittest || exit -1 + python -m nose tests/python/train || exit -1 + python3 -m nose tests/python/unittest || exit -1 + # python3 -m nose tests/python/train || exit -1 + else + nosetests tests/python/unittest || exit -1 + nosetests tests/python/train || exit -1 + nosetests3 tests/python/unittest || exit -1 + # nosetests3 tests/python/train || exit -1 + fi # if [ ${TRAVIS_OS_NAME} == "osx" ]; then # else - # nosetests tests/python/unittest || exit -1 - # nosetests tests/python/train || exit -1 # # TODO(mli) python3 error # # python3 --version # # rm -rf python/mxnet.egg-info # # cd python && python3 setup.py develop --user && cd .. || exit -1 - # # nosetests3 tests/python/unittest || exit -1 - # # nosetests3 tests/python/train || exit -1 # fi fi diff --git a/tests/travis/setup.sh b/tests/travis/setup.sh index ef4ee3810552..32def5fa6d47 100755 --- a/tests/travis/setup.sh +++ b/tests/travis/setup.sh @@ -6,13 +6,12 @@ if [ ${TRAVIS_OS_NAME} == "osx" ]; then brew info opencv brew install opencv brew install python3 + if [ ${TASK} == "python_test" ]; then + python -m pip install nose numpy --user `whoami` + python3 -m pip install nose numpy --user `whoami` + fi fi if [ ${TASK} == "lint" ]; then pip install cpplint pylint graphviz --user `whoami` fi - -if [ ${TASK} == "python_test" ]; then - python -m pip install nose numpy --user `whoami` - python3 -m pip install nose numpy --user `whoami` -fi From cd16d3cfcaca25cf1e8db7c586076882ab978a8f Mon Sep 17 00:00:00 2001 From: muli Date: Fri, 9 Oct 2015 01:08:02 -0400 Subject: [PATCH 19/24] [travis] fix cache --- tests/travis/run_test.sh | 25 ++----------------------- 1 file changed, 2 insertions(+), 23 deletions(-) diff --git a/tests/travis/run_test.sh b/tests/travis/run_test.sh index 225c6fdfe18a..923afaa788a2 100755 --- a/tests/travis/run_test.sh +++ b/tests/travis/run_test.sh @@ -1,13 +1,5 @@ #!/bin/bash - # - export PYTHONPATH=${PYTHONPATH}:${PWD}/python -python --version -${CXX} --version - -echo ${TASK} -echo ${TRAVIS_OS_NAME} -echo ${CXX} - if [ ${TASK} == "lint" ]; then make lint exit $? @@ -56,12 +48,9 @@ fi if [ ${TASK} == "python_test" ]; then make all || exit -1 # use cached dir for storing data - rm -rf tests/python/data + rm -rf ${PWD}/data mkdir -p ${CACHE_PREFIX}/data - ln -s ${CACHE_PREFIX}/data tests/python/data - - # python --version - # cd python && python setup.py develop --user && cd .. || exit -1 + ln -s ${CACHE_PREFIX}/data ${PWD}/data if [ ${TRAVIS_OS_NAME} == "osx" ]; then python -m nose tests/python/unittest || exit -1 @@ -74,14 +63,4 @@ if [ ${TASK} == "python_test" ]; then nosetests3 tests/python/unittest || exit -1 # nosetests3 tests/python/train || exit -1 fi - - # if [ ${TRAVIS_OS_NAME} == "osx" ]; then - # else - - - # # TODO(mli) python3 error - # # python3 --version - # # rm -rf python/mxnet.egg-info - # # cd python && python3 setup.py develop --user && cd .. || exit -1 - # fi fi From a1fde1c58842b76ad656c35bda70da9cad421f46 Mon Sep 17 00:00:00 2001 From: muli Date: Fri, 9 Oct 2015 15:17:31 -0400 Subject: [PATCH 20/24] [travis] update --- .travis.yml | 20 ++++++++++---------- tests/travis/run_test.sh | 1 + 2 files changed, 11 insertions(+), 10 deletions(-) diff --git a/.travis.yml b/.travis.yml index 3c2b841888a0..8bcec0e177cd 100644 --- a/.travis.yml +++ b/.travis.yml @@ -8,15 +8,15 @@ os: - osx env: - # # code analysis - # - TASK=lint - # # test doc build - # - TASK=doc - # # build mxnet.so with CUDA - # - TASK=build - # # run tests/cpp - # - TASK=cpp_test - # # run tests/python + # code analysis + - TASK=lint + # test doc build + - TASK=doc + # build mxnet.so with CUDA + - TASK=build + # run tests/cpp + - TASK=cpp_test + # run tests/python - TASK=python_test # TODO, R test, distributed test, clang, more g++ versions @@ -73,4 +73,4 @@ notifications: email: on_success: change on_failure: always -# slack: dmlc:NmroCzntCiWOuxUZpii40USd + slack: dmlc:NmroCzntCiWOuxUZpii40USd diff --git a/tests/travis/run_test.sh b/tests/travis/run_test.sh index 923afaa788a2..c28ccdb78df6 100755 --- a/tests/travis/run_test.sh +++ b/tests/travis/run_test.sh @@ -63,4 +63,5 @@ if [ ${TASK} == "python_test" ]; then nosetests3 tests/python/unittest || exit -1 # nosetests3 tests/python/train || exit -1 fi + exit 0 fi From 11818fe4cbec22582b25c700d0f658184399c783 Mon Sep 17 00:00:00 2001 From: muli Date: Fri, 9 Oct 2015 15:21:02 -0400 Subject: [PATCH 21/24] [travis] update --- tests/travis/setup.sh | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tests/travis/setup.sh b/tests/travis/setup.sh index 32def5fa6d47..526e71e357d2 100755 --- a/tests/travis/setup.sh +++ b/tests/travis/setup.sh @@ -13,5 +13,9 @@ if [ ${TRAVIS_OS_NAME} == "osx" ]; then fi if [ ${TASK} == "lint" ]; then - pip install cpplint pylint graphviz --user `whoami` + pip install cpplint pylint --user `whoami` +fi + +if [ ${TASK} == "doc" ]; then + pip install graphviz --user `whoami` fi From ea6e44eaaa1d7ccddb00beb9822e4f835eccd8bb Mon Sep 17 00:00:00 2001 From: muli Date: Fri, 9 Oct 2015 15:52:57 -0400 Subject: [PATCH 22/24] [travis] remove setup_osx.sh --- tests/travis/setup_osx.sh | 34 ---------------------------------- 1 file changed, 34 deletions(-) delete mode 100755 tests/travis/setup_osx.sh diff --git a/tests/travis/setup_osx.sh b/tests/travis/setup_osx.sh deleted file mode 100755 index a35e67e761ca..000000000000 --- a/tests/travis/setup_osx.sh +++ /dev/null @@ -1,34 +0,0 @@ -#!/bin/bash - -brew update -brew tap homebrew/science -brew info opencv -# brew install graphviz -brew install opencv -brew install python3 - -# TODO(mli) TASK are not correct... -# if [ ${TASK} == "python-package3" ]; then -# wget -O conda.sh https://repo.continuum.io/miniconda/Miniconda3-latest-MacOSX-x86_64.sh -# else -# wget -O conda.sh https://repo.continuum.io/miniconda/Miniconda-latest-MacOSX-x86_64.sh -# fi - - -# bash conda.sh -b -p $HOME/miniconda -# export PATH="$HOME/miniconda/bin:$PATH" -# hash -r -# conda config --set always_yes yes --set changeps1 no -# conda update -q conda -# # Useful for debugging any issues with conda -# conda info -a - -# if [ ${TASK} == "package3" ]; then -# conda create -n myenv python=3.4 -# else -# conda create -n myenv python=2.7 -# fi -# source activate myenv -# conda install numpy scipy matplotlib nose - -# python -m pip install nose numpy --user `whoami` From 061754120cd9008e3273ba3f53c356aa05718f6e Mon Sep 17 00:00:00 2001 From: muli Date: Fri, 9 Oct 2015 15:58:10 -0400 Subject: [PATCH 23/24] [travis] install graphivz --- .travis.yml | 2 +- tests/travis/setup.sh | 4 ---- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/.travis.yml b/.travis.yml index 8bcec0e177cd..313c4ec2a52f 100644 --- a/.travis.yml +++ b/.travis.yml @@ -48,7 +48,7 @@ addons: - python3-numpy - python3-dev - python3-nose - + - graphviz before_install: - export NVCC_PREFIX=${HOME} diff --git a/tests/travis/setup.sh b/tests/travis/setup.sh index 526e71e357d2..1f465f61e4cc 100755 --- a/tests/travis/setup.sh +++ b/tests/travis/setup.sh @@ -15,7 +15,3 @@ fi if [ ${TASK} == "lint" ]; then pip install cpplint pylint --user `whoami` fi - -if [ ${TASK} == "doc" ]; then - pip install graphviz --user `whoami` -fi From 09a0e3488e8ef30fcbcbe221b141db816d01e135 Mon Sep 17 00:00:00 2001 From: muli Date: Fri, 9 Oct 2015 17:20:49 -0400 Subject: [PATCH 24/24] [travis] train by python3 --- tests/travis/run_test.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/travis/run_test.sh b/tests/travis/run_test.sh index c28ccdb78df6..a0ebcd9edfcd 100755 --- a/tests/travis/run_test.sh +++ b/tests/travis/run_test.sh @@ -54,14 +54,14 @@ if [ ${TASK} == "python_test" ]; then if [ ${TRAVIS_OS_NAME} == "osx" ]; then python -m nose tests/python/unittest || exit -1 - python -m nose tests/python/train || exit -1 + # python -m nose tests/python/train || exit -1 python3 -m nose tests/python/unittest || exit -1 - # python3 -m nose tests/python/train || exit -1 + python3 -m nose tests/python/train || exit -1 else nosetests tests/python/unittest || exit -1 - nosetests tests/python/train || exit -1 + # nosetests tests/python/train || exit -1 nosetests3 tests/python/unittest || exit -1 - # nosetests3 tests/python/train || exit -1 + nosetests3 tests/python/train || exit -1 fi exit 0 fi