I'm running Ubuntu 16.04, CUDA Toolkit 7.5 and cudNN. The mnist demo works fine with gpus=0 but I get a segfault running the neural-style example. I used gdb to get this backtrace:
INFO:root:start training arguments Namespace(content_image='input/IMG_4343.jpg', content_weight=10, gpu=0, lr=0.001, max_long_edge=600, max_num_epochs=1000, model='vgg19', output='output/out.jpg', remove_noise=0.02, save_epochs=50, stop_eps=0.005, style_image='input/starry_night.jpg', style_weight=1, tv_weight=0.01)
Thread 9 "python" received signal SIGSEGV, Segmentation fault.
[Switching to Thread 0x7fffad01e700 (LWP 10535)]
0x00007fffebe2756b in std::_Rb_tree<dmlc::parameter::FieldAccessEntry*, dmlc::parameter::FieldAccessEntry*, std::_Identitydmlc::parameter::FieldAccessEntry*, std::lessdmlc::parameter::FieldAccessEntry*, std::allocatordmlc::parameter::FieldAccessEntry* >::_M_lower_bound (this=, __k=@0x28: ,
__y=0x7fffad01cd88, __x=0x0) at /usr/include/c++/4.9/bits/stl_tree.h:1276
1276 while (__x != 0)
(gdb) backtrace
#0 0x00007fffebe2756b in std::_Rb_tree<dmlc::parameter::FieldAccessEntry*, dmlc::parameter::FieldAccessEntry*, std::_Identitydmlc::parameter::FieldAccessEntry*, std::lessdmlc::parameter::FieldAccessEntry*, std::allocatordmlc::parameter::FieldAccessEntry* >::_M_lower_bound (this=, __k=@0x28: ,
__y=0x7fffad01cd88, __x=0x0) at /usr/include/c++/4.9/bits/stl_tree.h:1276
#1 std::_Rb_tree<dmlc::parameter::FieldAccessEntry*, dmlc::parameter::FieldAccessEntry*, std::_Identitydmlc::parameter::FieldAccessEntry*, std::lessdmlc::parameter::FieldAccessEntry*, std::allocatordmlc::parameter::FieldAccessEntry* >::find (__k=@0x28: , this=0x7fffad01cd80)
at /usr/include/c++/4.9/bits/stl_tree.h:1926
#2 std::set<dmlc::parameter::FieldAccessEntry*, std::lessdmlc::parameter::FieldAccessEntry*, std::allocatordmlc::parameter::FieldAccessEntry* >::count (__x=@0x28: , this=0x7fffad01cd80)
at /usr/include/c++/4.9/bits/stl_set.h:684
#3 dmlc::parameter::ParamManager::RunInit<__gnu_cxx::__normal_iteratorstd::pair<std::basic_string<char, std::basic_string > const*, std::vectorstd::pair<std::basic_string<char, std::basic_string > > > > (
this=0x7fffed7b1020 <mxnet::op::ReduceAxisParam::__MANAGER__()::inst>, head=head@entry=0x7fffad01cfd0,
begin=begin@entry=<error reading variable: Cannot access memory at address 0x0>,
end=end@entry=<error reading variable: Cannot access memory at address 0x0>, unknown_args=0x0)
at /home/tbullock/mxnet/dmlc-core/include/dmlc/./parameter.h:380
#4 0x00007fffebe7dc16 in dmlc::Parametermxnet::op::ReduceAxisParam::Initstd::vector<std::pair<std::basic_string<char, std::basic_string > > > (kwargs=std::vector of length 0, capacity 0, this=0x7fffad01cfd0)
at /home/tbullock/mxnet/dmlc-core/include/dmlc/./parameter.h:116
#5 mxnet::op::ReduceAxis<mshadow::gpu, mshadow::red::sum> (src=..., env=..., ret=0x7fffad01d060,
req=mxnet::kWriteTo, ctx=...) at src/operator/./broadcast_reduce_op-inl.h:282
#6 0x00007fffeb8ff027 in mxnet::op::SimpleUnaryOperator::Forward(mxnet::OpContext const&, std::vector<mshadow::TBlob, std::allocatormshadow::TBlob > const&, std::vector<mxnet::OpReqType, std::allocatormxnet::OpReqType > const&, std::vector<mshadow::TBlob, std::allocatormshadow::TBlob > const&, std::vector<mshadow::TBlob, std::allocatormshadow::TBlob > const&) ()
from /home/tbullock/anaconda3/envs/digits/lib/python2.7/site-packages/mxnet-0.7.0-py2.7.egg/mxnet/libmxnet.so
#7 0x00007fffebb9d1eb in mxnet::GraphExecutor::CreateCachedSegOpr(unsigned long, unsigned long)::{lambda(mxnet::RunContext, mxnet::engine::CallbackOnComplete)#1}::operator()(mxnet::RunContext, mxnet::engine::CallbackOnComplete) const () from /home/tbullock/anaconda3/envs/digits/lib/python2.7/site-packages/mxnet-0.7.0-py2.7.egg/mxnet/libmxnet.so
#8 0x00007fffebb9dc37 in std::_Function_handler<void (mxnet::RunContext, mxnet::engine::CallbackOnComplete), mxnet::GraphExecutor::CreateCachedSegOpr(unsigned long, unsigned long)::{lambda(mxnet::RunContext, mxnet::engine::CallbackOnComplete)#1}>::_M_invoke(std::_Any_data const&, mxnet::RunContext&&, mxnet::engine::CallbackOnComplete&&) ()
from /home/tbullock/anaconda3/envs/digits/lib/python2.7/site-packages/mxnet-0.7.0-py2.7.egg/mxnet/libmxnet.so
#9 0x00007fffebb876d4 in mxnet::engine::ThreadedEngine::ExecuteOprBlock(mxnet::RunContext, mxnet::engine::OprBlock*) () from /home/tbullock/anaconda3/envs/digits/lib/python2.7/site-packages/mxnet-0.7.0-py2.7.egg/mxnet/libmxnet.so
#10 0x00007fffebb88ec8 in std::_Function_handler<void (), mxnet::engine::ThreadedEnginePerDevice::PushToExecute(mxnet::engine::OprBlock*, bool)::{lambda()#3}::operator()() const::{lambda()#1}>::_M_invoke(std::_Any_data const&) ()
from /home/tbullock/anaconda3/envs/digits/lib/python2.7/site-packages/mxnet-0.7.0-py2.7.egg/mxnet/libmxnet.so
#11 0x00007fffe1401c80 in ?? () from /usr/lib/x86_64-linux-gnu/libstdc++.so.6
#12 0x00007ffff77c76fa in start_thread (arg=0x7fffad01e700) at pthread_create.c:333
#13 0x00007ffff6dedb5d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109
I'm running Ubuntu 16.04, CUDA Toolkit 7.5 and cudNN. The mnist demo works fine with gpus=0 but I get a segfault running the neural-style example. I used gdb to get this backtrace:
INFO:root:start training arguments Namespace(content_image='input/IMG_4343.jpg', content_weight=10, gpu=0, lr=0.001, max_long_edge=600, max_num_epochs=1000, model='vgg19', output='output/out.jpg', remove_noise=0.02, save_epochs=50, stop_eps=0.005, style_image='input/starry_night.jpg', style_weight=1, tv_weight=0.01)
Thread 9 "python" received signal SIGSEGV, Segmentation fault.
[Switching to Thread 0x7fffad01e700 (LWP 10535)]
0x00007fffebe2756b in std::_Rb_tree<dmlc::parameter::FieldAccessEntry*, dmlc::parameter::FieldAccessEntry*, std::_Identitydmlc::parameter::FieldAccessEntry*, std::lessdmlc::parameter::FieldAccessEntry*, std::allocatordmlc::parameter::FieldAccessEntry* >::_M_lower_bound (this=, __k=@0x28: ,
__y=0x7fffad01cd88, __x=0x0) at /usr/include/c++/4.9/bits/stl_tree.h:1276
1276 while (__x != 0)
(gdb) backtrace
#0 0x00007fffebe2756b in std::_Rb_tree<dmlc::parameter::FieldAccessEntry*, dmlc::parameter::FieldAccessEntry*, std::_Identitydmlc::parameter::FieldAccessEntry*, std::lessdmlc::parameter::FieldAccessEntry*, std::allocatordmlc::parameter::FieldAccessEntry* >::_M_lower_bound (this=, __k=@0x28: ,
#1 std::_Rb_tree<dmlc::parameter::FieldAccessEntry*, dmlc::parameter::FieldAccessEntry*, std::_Identitydmlc::parameter::FieldAccessEntry*, std::lessdmlc::parameter::FieldAccessEntry*, std::allocatordmlc::parameter::FieldAccessEntry* >::find (__k=@0x28: , this=0x7fffad01cd80)
#2 std::set<dmlc::parameter::FieldAccessEntry*, std::lessdmlc::parameter::FieldAccessEntry*, std::allocatordmlc::parameter::FieldAccessEntry* >::count (__x=@0x28: , this=0x7fffad01cd80)
#3 dmlc::parameter::ParamManager::RunInit<__gnu_cxx::__normal_iteratorstd::pair<std::basic_string<char, std::basic_string > const*, std::vectorstd::pair<std::basic_string<char, std::basic_string > > > > (
#4 0x00007fffebe7dc16 in dmlc::Parametermxnet::op::ReduceAxisParam::Initstd::vector<std::pair<std::basic_string<char, std::basic_string > > > (kwargs=std::vector of length 0, capacity 0, this=0x7fffad01cfd0)
#5 mxnet::op::ReduceAxis<mshadow::gpu, mshadow::red::sum> (src=..., env=..., ret=0x7fffad01d060,
#6 0x00007fffeb8ff027 in mxnet::op::SimpleUnaryOperator::Forward(mxnet::OpContext const&, std::vector<mshadow::TBlob, std::allocatormshadow::TBlob > const&, std::vector<mxnet::OpReqType, std::allocatormxnet::OpReqType > const&, std::vector<mshadow::TBlob, std::allocatormshadow::TBlob > const&, std::vector<mshadow::TBlob, std::allocatormshadow::TBlob > const&) ()
from /home/tbullock/anaconda3/envs/digits/lib/python2.7/site-packages/mxnet-0.7.0-py2.7.egg/mxnet/libmxnet.so
#7 0x00007fffebb9d1eb in mxnet::GraphExecutor::CreateCachedSegOpr(unsigned long, unsigned long)::{lambda(mxnet::RunContext, mxnet::engine::CallbackOnComplete)#1}::operator()(mxnet::RunContext, mxnet::engine::CallbackOnComplete) const () from /home/tbullock/anaconda3/envs/digits/lib/python2.7/site-packages/mxnet-0.7.0-py2.7.egg/mxnet/libmxnet.so
#8 0x00007fffebb9dc37 in std::_Function_handler<void (mxnet::RunContext, mxnet::engine::CallbackOnComplete), mxnet::GraphExecutor::CreateCachedSegOpr(unsigned long, unsigned long)::{lambda(mxnet::RunContext, mxnet::engine::CallbackOnComplete)#1}>::_M_invoke(std::_Any_data const&, mxnet::RunContext&&, mxnet::engine::CallbackOnComplete&&) ()
from /home/tbullock/anaconda3/envs/digits/lib/python2.7/site-packages/mxnet-0.7.0-py2.7.egg/mxnet/libmxnet.so
#9 0x00007fffebb876d4 in mxnet::engine::ThreadedEngine::ExecuteOprBlock(mxnet::RunContext, mxnet::engine::OprBlock*) () from /home/tbullock/anaconda3/envs/digits/lib/python2.7/site-packages/mxnet-0.7.0-py2.7.egg/mxnet/libmxnet.so
#10 0x00007fffebb88ec8 in std::_Function_handler<void (), mxnet::engine::ThreadedEnginePerDevice::PushToExecute(mxnet::engine::OprBlock*, bool)::{lambda()#3}::operator()() const::{lambda()#1}>::_M_invoke(std::_Any_data const&) ()
from /home/tbullock/anaconda3/envs/digits/lib/python2.7/site-packages/mxnet-0.7.0-py2.7.egg/mxnet/libmxnet.so
#11 0x00007fffe1401c80 in ?? () from /usr/lib/x86_64-linux-gnu/libstdc++.so.6
#12 0x00007ffff77c76fa in start_thread (arg=0x7fffad01e700) at pthread_create.c:333
#13 0x00007ffff6dedb5d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109