merge bitcoin#19817, bitcoin#20470, bitcoin#22993: macOS C++17 bump

.gitignore

@@ -74,6 +74,7 @@ libconftest.dylib*
 *.log
 *.trs
 *.dmg
+*.iso
 
 *.json.h
 *.raw.h

.gitlab-ci.yml

@@ -37,13 +37,13 @@ builder-image:
   image: $CI_REGISTRY_IMAGE:builder-$CI_COMMIT_REF_SLUG
   variables:
     SDK_URL: https://bitcoincore.org/depends-sources/sdks
-    XCODE_VERSION: 11.3.1
-    XCODE_BUILD_ID: 11C505
+    XCODE_VERSION: "12.1"
+    XCODE_BUILD_ID: 12A7403
     MAKEJOBS: -j4
   before_script:
     - echo HOST=$HOST
     - |
-      if [ "$HOST" = "x86_64-apple-darwin18" ]; then
+      if [ "$HOST" = "x86_64-apple-darwin19" ]; then
         mkdir -p depends/SDKs
         mkdir -p depends/sdk-sources
         OSX_SDK_BASENAME="Xcode-${XCODE_VERSION}-${XCODE_BUILD_ID}-extracted-SDK-with-libcxx-headers.tar.gz"
@@ -194,12 +194,12 @@ x86_64-unknown-linux-gnu-release:
     HOST: x86_64-unknown-linux-gnu
     DEP_OPTS: "NO_UPNP=1"
 
-x86_64-apple-darwin18:
+x86_64-apple-darwin19:
   extends:
     - .build-depends-template
     - .skip-in-fast-mode-template
   variables:
-    HOST: x86_64-apple-darwin18
+    HOST: x86_64-apple-darwin19
 
 ###
 
@@ -265,7 +265,7 @@ mac-build:
     - .build-template
     - .skip-in-fast-mode-template
   needs:
-    - x86_64-apple-darwin18
+    - x86_64-apple-darwin19
   variables:
     BUILD_TARGET: mac
 

.travis.yml

@@ -296,7 +296,7 @@ after_success:
     - stage: test
       name: 'macOS 10.12  [GOAL: deploy]'
       env: >-
-        HOST=x86_64-apple-darwin18
+        HOST=x86_64-apple-darwin19
         PACKAGES="cmake imagemagick libcap-dev librsvg2-bin libz-dev libbz2-dev libtiff-tools python3-dev python3-setuptools"
         export XCODE_VERSION=11.3.1
         export XCODE_BUILD_ID=11C505

CMakeLists.txt

@@ -20,7 +20,7 @@ include_directories(
 if(UNIX AND NOT APPLE)
     set(DEPENDS_PREFIX depends/x86_64-pc-linux-gnu)
 elseif(APPLE)
-    set(DEPENDS_PREFIX depends/x86_64-apple-darwin18)
+    set(DEPENDS_PREFIX depends/x86_64-apple-darwin19)
 elseif(WIN32)
     set(DEPENDS_PREFIX depends/x86_64-w64-mingw32)
 endif()

Makefile.am

@@ -34,6 +34,7 @@ space := $(empty) $(empty)
 OSX_APP=Dash-Qt.app
 OSX_VOLNAME = $(subst $(space),-,$(PACKAGE_NAME))
 OSX_DMG = $(OSX_VOLNAME).dmg
+OSX_TEMP_ISO = $(OSX_DMG:.dmg=).temp.iso
 OSX_BACKGROUND_SVG=background.svg
 OSX_BACKGROUND_IMAGE=background.tiff
 OSX_BACKGROUND_IMAGE_DPIS=36 72
@@ -144,8 +145,12 @@ $(APP_DIST_DIR)/Applications:
 
 $(APP_DIST_EXTRAS): $(APP_DIST_DIR)/$(OSX_APP)/Contents/MacOS/Dash-Qt
 
-$(OSX_DMG): $(APP_DIST_EXTRAS)
-	$(GENISOIMAGE) -no-cache-inodes -D -l -probe -V "$(OSX_VOLNAME)" -no-pad -r -dir-mode 0755 -apple -o $@ dist
+.INTERMEDIATE: $(OSX_TEMP_ISO)
+$(OSX_TEMP_ISO): $(APP_DIST_EXTRAS)
+	$(XORRISOFS) -D -l -V "$(OSX_VOLNAME)" -no-pad -r -dir-mode 0755 -o $@ dist
+
+$(OSX_DMG): $(OSX_TEMP_ISO)
+	$(DMG) dmg "$<" "$@"
 
 dpi%.$(OSX_BACKGROUND_IMAGE): contrib/macdeploy/$(OSX_BACKGROUND_SVG)
 	sed 's/PACKAGE_NAME/$(PACKAGE_NAME)/' < "$<" | $(RSVG_CONVERT) -f png -d $* -p $* | $(IMAGEMAGICK_CONVERT) - $@

ci/Dockerfile.builder

@@ -38,7 +38,7 @@ RUN dpkg --add-architecture i386
 RUN apt-get update && apt-get install $APT_ARGS g++-7-multilib && rm -rf /var/lib/apt/lists/*
 RUN apt-get update && apt-get install $APT_ARGS g++-arm-linux-gnueabihf && rm -rf /var/lib/apt/lists/*
 RUN apt-get update && apt-get install $APT_ARGS g++-mingw-w64-x86-64 && rm -rf /var/lib/apt/lists/*
-RUN apt-get update && apt-get install $APT_ARGS wine-stable wine32 wine64 bc nsis && rm -rf /var/lib/apt/lists/*
+RUN apt-get update && apt-get install $APT_ARGS wine-stable wine32 wine64 bc nsis xorriso && rm -rf /var/lib/apt/lists/*
 RUN apt-get update && apt-get install $APT_ARGS python3-zmq && rm -rf /var/lib/apt/lists/*
 RUN apt-get update && apt-get install $APT_ARGS shellcheck cppcheck && rm -rf /var/lib/apt/lists/*
 RUN apt-get update && apt-get install $APT_ARGS imagemagick libcap-dev librsvg2-bin libz-dev libbz2-dev libtiff-tools && rm -rf /var/lib/apt/lists/*

ci/matrix.sh

@@ -76,7 +76,7 @@ elif [ "$BUILD_TARGET" = "linux64_release" ]; then
   export BITCOIN_CONFIG="--enable-zmq --enable-glibc-back-compat --enable-reduce-exports"
   export PYZMQ=true
 elif [ "$BUILD_TARGET" = "mac" ]; then
-  export HOST=x86_64-apple-darwin18
+  export HOST=x86_64-apple-darwin19
   export BITCOIN_CONFIG="--enable-gui --enable-reduce-exports --disable-miner --enable-werror"
   export XCODE_VERSION=11.3.1
   export XCODE_BUILD_ID=11C505

configure.ac

@@ -645,7 +645,8 @@ case $host in
            AC_PATH_TOOL([DSYMUTIL], [dsymutil], dsymutil)
            AC_PATH_TOOL([INSTALLNAMETOOL], [install_name_tool], install_name_tool)
            AC_PATH_TOOL([OTOOL], [otool], otool)
-           AC_PATH_PROGS([GENISOIMAGE], [genisoimage mkisofs],genisoimage)
+           AC_PATH_PROGS([XORRISOFS], [xorrisofs], xorrisofs)
+           AC_PATH_PROGS([DMG], [dmg], dmg)
            AC_PATH_PROGS([RSVG_CONVERT], [rsvg-convert rsvg],rsvg-convert)
            AC_PATH_PROGS([IMAGEMAGICK_CONVERT], [convert],convert)
            AC_PATH_PROGS([TIFFCP], [tiffcp],tiffcp)

contrib/gitian-build.py

@@ -71,8 +71,8 @@ def build():
 
     if args.macos:
         print('\nCompiling ' + args.version + ' MacOS')
-        subprocess.check_call(['wget', '-N', '-P', 'inputs', 'https://bitcoincore.org/depends-sources/sdks/Xcode-11.3.1-11C505-extracted-SDK-with-libcxx-headers.tar.gz'])
-        subprocess.check_output(["echo '436df6dfc7073365d12f8ef6c1fdb060777c720602cc67c2dcf9a59d94290e38 inputs/Xcode-11.3.1-11C505-extracted-SDK-with-libcxx-headers.tar.gz' | sha256sum -c"], shell=True)
+        subprocess.check_call(['wget', '-N', '-P', 'inputs', 'https://bitcoincore.org/depends-sources/sdks/Xcode-12.1-12A7403-extracted-SDK-with-libcxx-headers.tar.gz'])
+        subprocess.check_output(["echo 'be17f48fd0b08fb4dcd229f55a6ae48d9f781d210839b4ea313ef17dd12d6ea5 inputs/Xcode-12.1-12A7403-extracted-SDK-with-libcxx-headers.tar.gz' | sha256sum -c"], shell=True)
         subprocess.check_call(['bin/gbuild', '-j', args.jobs, '-m', args.memory, '--commit', 'dash='+args.commit, '--url', 'dash='+args.url, '../dash/contrib/gitian-descriptors/gitian-osx.yml'])
         subprocess.check_call(['bin/gsign', '-p', args.sign_prog, '--signer', args.signer, '--release', args.version+'-osx-unsigned', '--destination', '../gitian.sigs/', '../dash/contrib/gitian-descriptors/gitian-osx.yml'])
         subprocess.check_call('mv build/out/dashcore-*-osx-unsigned.tar.gz inputs/', shell=True)

contrib/gitian-descriptors/gitian-osx-signer.yml

@@ -7,6 +7,7 @@ architectures:
 - "amd64"
 packages:
 - "faketime"
+- "xorriso"
 remotes:
 - "url": "https://github.com/dashpay/dash-detached-sigs.git"
   "dir": "signature"
@@ -18,7 +19,7 @@ script: |
   WRAP_DIR=$HOME/wrapped
   mkdir -p ${WRAP_DIR}
   export PATH="$PWD":$PATH
-  FAKETIME_PROGS="dmg genisoimage"
+  FAKETIME_PROGS="dmg xorrisofs"
 
   # Create global faketime wrappers
   for prog in ${FAKETIME_PROGS}; do
@@ -36,5 +37,5 @@ script: |
   tar -xf ${UNSIGNED}
   OSX_VOLNAME="$(cat osx_volname)"
   ./detached-sig-apply.sh ${UNSIGNED} signature/osx
-  ${WRAP_DIR}/genisoimage -no-cache-inodes -D -l -probe -V "${OSX_VOLNAME}" -no-pad -r -dir-mode 0755 -apple -o uncompressed.dmg signed-app
+  ${WRAP_DIR}/xorrisofs -D -l -V "${OSX_VOLNAME}" -no-pad -r -dir-mode 0755 -o uncompressed.dmg signed-app
   ${WRAP_DIR}/dmg dmg uncompressed.dmg ${OUTDIR}/${SIGNED}

contrib/gitian-descriptors/gitian-osx.yml

@@ -29,19 +29,20 @@ packages:
 - "fonts-tuffy"
 - "ccache"
 - "cmake"
+- "xorriso"
 remotes:
 - "url": "https://github.com/dashpay/dash.git"
   "dir": "dash"
 files:
-- "Xcode-11.3.1-11C505-extracted-SDK-with-libcxx-headers.tar.gz"
+- "Xcode-12.1-12A7403-extracted-SDK-with-libcxx-headers.tar.gz"
 script: |
   set -e -o pipefail
 
   WRAP_DIR=$HOME/wrapped
-  HOSTS="x86_64-apple-darwin18"
-  CONFIGFLAGS="--enable-reduce-exports --disable-miner --disable-bench --disable-gui-tests GENISOIMAGE=$WRAP_DIR/genisoimage --enable-crash-hooks"
+  HOSTS="x86_64-apple-darwin19"
+  CONFIGFLAGS="--enable-reduce-exports --disable-miner --disable-bench --disable-gui-tests XORRISOFS=${WRAP_DIR}/xorrisofs DMG=${WRAP_DIR}/dmg --enable-crash-hooks"
   FAKETIME_HOST_PROGS=""
-  FAKETIME_PROGS="ar ranlib date dmg genisoimage"
+  FAKETIME_PROGS="ar ranlib date dmg xorrisofs"
 
   export QT_RCC_TEST=1
   export QT_RCC_SOURCE_DATE_OVERRIDE=1
@@ -109,7 +110,7 @@ script: |
   BASEPREFIX="${PWD}/depends"
 
   mkdir -p ${BASEPREFIX}/SDKs
-  tar -C ${BASEPREFIX}/SDKs -xf ${BUILD_DIR}/Xcode-11.3.1-11C505-extracted-SDK-with-libcxx-headers.tar.gz
+  tar -C ${BASEPREFIX}/SDKs -xf ${BUILD_DIR}/Xcode-12.1-12A7403-extracted-SDK-with-libcxx-headers.tar.gz
 
   # Build dependencies for each host
   for i in $HOSTS; do
@@ -161,21 +162,19 @@ script: |
 
     make osx_volname
     make deploydir
-    OSX_VOLNAME="$(cat osx_volname)"
     mkdir -p unsigned-app-${i}
     cp osx_volname unsigned-app-${i}/
     cp contrib/macdeploy/detached-sig-apply.sh unsigned-app-${i}
     cp contrib/macdeploy/detached-sig-create.sh unsigned-app-${i}
-    cp ${BASEPREFIX}/${i}/native/bin/dmg ${BASEPREFIX}/${i}/native/bin/genisoimage unsigned-app-${i}
+    cp ${BASEPREFIX}/${i}/native/bin/dmg unsigned-app-${i}
     cp ${BASEPREFIX}/${i}/native/bin/${i}-codesign_allocate unsigned-app-${i}/codesign_allocate
     cp ${BASEPREFIX}/${i}/native/bin/${i}-pagestuff unsigned-app-${i}/pagestuff
     mv dist unsigned-app-${i}
     pushd unsigned-app-${i}
     find . | sort | tar --no-recursion --mode='u+rw,go+r-w,a+X' --owner=0 --group=0 -c -T - | gzip -9n > ${OUTDIR}/${DISTNAME}-osx-unsigned.tar.gz
     popd
 
-    make deploy
-    ${WRAP_DIR}/dmg dmg "${OSX_VOLNAME}.dmg" ${OUTDIR}/${DISTNAME}-osx-unsigned.dmg
+    make deploy OSX_DMG="${OUTDIR}/${DISTNAME}-osx-unsigned.dmg"
 
     cd installed
     find . -name "lib*.la" -delete
@@ -188,8 +187,8 @@ script: |
   done
   mkdir -p $OUTDIR/src
   mv $SOURCEDIST $OUTDIR/src
-  mv ${OUTDIR}/${DISTNAME}-x86_64-apple-darwin18.tar.gz ${OUTDIR}/${DISTNAME}-osx64.tar.gz
-  mv ${OUTDIR}/${DISTNAME}-x86_64-apple-darwin18-debug.tar.gz ${OUTDIR}/${DISTNAME}-osx64-debug.tar.gz
+  mv ${OUTDIR}/${DISTNAME}-x86_64-apple-darwin19.tar.gz ${OUTDIR}/${DISTNAME}-osx64.tar.gz
+  mv ${OUTDIR}/${DISTNAME}-x86_64-apple-darwin19-debug.tar.gz ${OUTDIR}/${DISTNAME}-osx64-debug.tar.gz
 
   # Compress ccache (otherwise the assert file will get too huge)
   if [ "$CCACHE_DIR" != "" ]; then

contrib/macdeploy/README.md

@@ -17,9 +17,9 @@ When complete, it will have produced `Dash-Qt.dmg`.
 ### Step 1: Obtaining `Xcode.app`
 
 Our current macOS SDK
-(`Xcode-11.3.1-11C505-extracted-SDK-with-libcxx-headers.tar.gz`) can be
+(`Xcode-12.1-12A7403-extracted-SDK-with-libcxx-headers.tar.gz`) can be
 extracted from
-[Xcode_11.3.1.xip](https://download.developer.apple.com/Developer_Tools/Xcode_11.3.1/Xcode_11.3.1.xip).
+[Xcode_12.1.xip](https://download.developer.apple.com/Developer_Tools/Xcode_12.1/Xcode_12.1.xip).
 An Apple ID is needed to download this.
 
 After Xcode version 7.x, Apple started shipping the `Xcode.app` in a `.xip`
@@ -31,25 +31,25 @@ approach (tested on Debian Buster) is outlined below:
 apt install cpio
 git clone https://github.com/bitcoin-core/apple-sdk-tools.git
 
-# Unpack Xcode_11.3.1.xip and place the resulting Xcode.app in your current
+# Unpack Xcode_12.1.xip and place the resulting Xcode.app in your current
 # working directory
-python3 apple-sdk-tools/extract_xcode.py -f Xcode_11.3.1.xip | cpio -d -i
+python3 apple-sdk-tools/extract_xcode.py -f Xcode_12.1.xip | cpio -d -i
 ```
 
 On macOS the process is more straightforward:
 
 ```bash
-xip -x Xcode_11.3.1.xip
+xip -x Xcode_12.1.xip
 ```
 
-### Step 2: Generating `Xcode-11.3.1-11C505-extracted-SDK-with-libcxx-headers.tar.gz` from `Xcode.app`
+### Step 2: Generating `Xcode-12.1-12A7403-extracted-SDK-with-libcxx-headers.tar.gz` from `Xcode.app`
 
-To generate `Xcode-11.3.1-11C505-extracted-SDK-with-libcxx-headers.tar.gz`, run
+To generate `Xcode-12.1-12A7403-extracted-SDK-with-libcxx-headers.tar.gz`, run
 the script [`gen-sdk`](./gen-sdk) with the path to `Xcode.app` (extracted in the
 previous stage) as the first argument.
 
 ```bash
-# Generate a Xcode-11.3.1-11C505-extracted-SDK-with-libcxx-headers.tar.gz from
+# Generate a Xcode-12.1-12A7403-extracted-SDK-with-libcxx-headers.tar.gz from
 # the supplied Xcode.app
 ./contrib/macdeploy/gen-sdk '/path/to/Xcode.app'
 ```
@@ -96,19 +96,15 @@ created using these tools. The build process has been designed to avoid includin
 SDK's files in Gitian's outputs. All interim tarballs are fully deterministic and may be freely
 redistributed.
 
-`genisoimage` is used to create the initial DMG. It is not deterministic as-is, so it has been
-patched. A system `genisoimage` will work fine, but it will not be deterministic because
-the file-order will change between invocations. The patch can be seen here: [cdrkit-deterministic.patch](https://github.com/bitcoin/bitcoin/blob/master/depends/patches/native_cdrkit/cdrkit-deterministic.patch).
-No effort was made to fix this cleanly, so it likely leaks memory badly, however it's only used for
-a single invocation, so that's no real concern.
+[`xorrisofs`](https://www.gnu.org/software/xorriso/) is used to create the DMG.
 
-`genisoimage` cannot compress DMGs, so afterwards, the DMG tool from the
-`libdmg-hfsplus` project is used to compress it. There are several bugs in this tool and its
-maintainer has seemingly abandoned the project.
+`xorrisofs` cannot compress DMGs, so afterwards, the DMG tool from the
+`libdmg-hfsplus` project is used to compress it. There are several bugs in this
+tool and its maintainer has seemingly abandoned the project.
 
 The DMG tool has the ability to create DMGs from scratch as well, but this functionality is
 broken. Only the compression feature is currently used. Ideally, the creation could be fixed
-and `genisoimage` would no longer be necessary.
+and `xorrisofs` would no longer be necessary.
 
 Background images and other features can be added to DMG files by inserting a
 `.DS_Store` before creation. This is generated by the script `contrib/macdeploy/custom_dsstore.py`.

depends/README.md

@@ -21,7 +21,7 @@ created. To use it for Dash:
 Common `host-platform-triplets` for cross compilation are:
 
 - `x86_64-w64-mingw32` for Win64
-- `x86_64-apple-darwin18` for macOS
+- `x86_64-apple-darwin19` for macOS
 - `arm-linux-gnueabihf` for Linux ARM 32 bit
 - `aarch64-linux-gnu` for Linux ARM 64 bit
 - `riscv32-linux-gnu` for Linux RISC-V 32 bit

depends/hosts/darwin.mk

@@ -1,8 +1,8 @@
-OSX_MIN_VERSION=10.14
-OSX_SDK_VERSION=10.15.1
-XCODE_VERSION=11.3.1
-XCODE_BUILD_ID=11C505
-LD64_VERSION=530
+OSX_MIN_VERSION=10.15
+OSX_SDK_VERSION=10.15.6
+XCODE_VERSION=12.1
+XCODE_BUILD_ID=12A7403
+LD64_VERSION=609
 
 OSX_SDK=$(SDK_PATH)/Xcode-$(XCODE_VERSION)-$(XCODE_BUILD_ID)-extracted-SDK-with-libcxx-headers
 
@@ -60,16 +60,11 @@ $(foreach TOOL,$(cctools_TOOLS),$(eval darwin_$(TOOL) = $$(build_prefix)/bin/$$(
 #         Explicitly point to our binaries (e.g. cctools) so that they are
 #         ensured to be found and preferred over other possibilities.
 #
-#     -stdlib=libc++ -nostdinc++ -Xclang -cxx-isystem$(OSX_SDK)/usr/include/c++/v1
+#     -stdlib=libc++ -stdlib++-isystem$(OSX_SDK)/usr/include/c++/v1
 #
 #         Forces clang to use the libc++ headers from our SDK and completely
 #         forget about the libc++ headers from the standard directories
 #
-#         TODO: Once we start requiring a clang version that has the
-#         -stdlib++-isystem<directory> flag first introduced here:
-#         https://reviews.llvm.org/D64089, we should use that instead. Read the
-#         differential summary there for more details.
-#
 #     -Xclang -*system<path_a> \
 #     -Xclang -*system<path_b> \
 #     -Xclang -*system<path_c> ...
@@ -109,8 +104,8 @@ darwin_CXX=env -u C_INCLUDE_PATH -u CPLUS_INCLUDE_PATH \
              $(clangxx_prog) --target=$(host) -mmacosx-version-min=$(OSX_MIN_VERSION) \
                -B$(build_prefix)/bin -mlinker-version=$(LD64_VERSION) \
                -isysroot$(OSX_SDK) \
-               -stdlib=libc++ -nostdinc++ \
-               -Xclang -cxx-isystem$(OSX_SDK)/usr/include/c++/v1 \
+               -stdlib=libc++ \
+               -stdlib++-isystem$(OSX_SDK)/usr/include/c++/v1 \
                -Xclang -internal-externc-isystem$(clang_resource_dir)/include \
                -Xclang -internal-externc-isystem$(OSX_SDK)/usr/include
 

depends/packages/bls-dash.mk

@@ -7,7 +7,7 @@ $(package)_build_subdir=build
 $(package)_sha256_hash=276c8573104e5f18bb5b9fd3ffd49585dda5ba5f6de2de74759dda8ca5a9deac
 $(package)_dependencies=gmp cmake
 $(package)_patches=gcc_alignment_cast.patch
-$(package)_darwin_triplet=x86_64-apple-darwin18
+$(package)_darwin_triplet=x86_64-apple-darwin19
 
 $(package)_relic_version=3a23142be0a5510a3aa93cd6c76fc59d3fc732a5
 $(package)_relic_download_path=https://github.com/relic-toolkit/relic/archive

depends/packages/native_cctools.mk

@@ -1,10 +1,9 @@
 package=native_cctools
-$(package)_version=55562e4073dea0fbfd0b20e0bf69ffe6390c7f97
+$(package)_version=2ef2e931cf641547eb8a68cfebde61003587c9fd
 $(package)_download_path=https://github.com/tpoechtrager/cctools-port/archive
 $(package)_file_name=$($(package)_version).tar.gz
-$(package)_sha256_hash=e51995a843533a3dac155dd0c71362dd471597a2d23f13dff194c6285362f875
+$(package)_sha256_hash=6b73269efdf5c58a070e7357b66ee760501388549d6a12b423723f45888b074b
 $(package)_build_subdir=cctools
-$(package)_patches=ld64_disable_threading.patch
 $(package)_dependencies=native_libtapi
 
 define $(package)_set_vars
@@ -17,10 +16,6 @@ define $(package)_set_vars
   $(package)_cxx=$(clangxx_prog)
 endef
 
-define $(package)_preprocess_cmds
-  patch -p1 < $($(package)_patch_dir)/ld64_disable_threading.patch
-endef
-
 define $(package)_config_cmds
   $($(package)_autoconf)
 endef