Merge #19205: script: previous_release.sh rewritten in python

9c34aff39309b8adc99d347e07b6ddb5366498e9 Remove previous_release.sh (Brian Liotti)
e1e5960e10a9329d9f55a3967d546ffbdd896030 script: Add previous_release.py (Brian Liotti)

Pull request description:

  Closes #18132

  Added functionality:
  1) checks file hash before untarring when using the binary download option

ACKs for top commit:
  fjahr:
    re-ACK 9c34aff39309b8adc99d347e07b6ddb5366498e9
  Sjors:
    tACK 9c34aff39309b8adc99d347e07b6ddb5366498e9

Tree-SHA512: 323f11828736a372a47f048592de8b027ddcd75b38f312dfc73f7b495d1e078bfeb384d9cdf434b3e70f2c6c0ce2da2df48e9a6460ac0e1967c6829a411c52d5

ci/test/05_before_script.sh

@@ -48,6 +48,6 @@ if [ -z "$NO_DEPENDS" ]; then
 fi
 if [ -n "$PREVIOUS_RELEASES_TO_DOWNLOAD" ]; then
   BEGIN_FOLD previous-versions
-  DOCKER_EXEC contrib/devtools/previous_release.sh -b -t "$PREVIOUS_RELEASES_DIR" "${PREVIOUS_RELEASES_TO_DOWNLOAD}"
+  DOCKER_EXEC contrib/devtools/previous_release.py -b -t "$PREVIOUS_RELEASES_DIR" "${PREVIOUS_RELEASES_TO_DOWNLOAD}"
   END_FOLD
 fi

contrib/devtools/previous_release.py

@@ -0,0 +1,222 @@
+#!/usr/bin/env python3
+#
+# Copyright (c) 2018-2020 The Bitcoin Core developers
+# Distributed under the MIT software license, see the accompanying
+# file COPYING or http://www.opensource.org/licenses/mit-license.php.
+#
+# Build previous releases.
+
+import argparse
+import contextlib
+from fnmatch import fnmatch
+import os
+from pathlib import Path
+import re
+import shutil
+import subprocess
+import sys
+import hashlib
+
+
+@contextlib.contextmanager
+def pushd(new_dir) -> None:
+    previous_dir = os.getcwd()
+    os.chdir(new_dir)
+    try:
+        yield
+    finally:
+        os.chdir(previous_dir)
+
+
+def download_binary(tag, args) -> int:
+    if Path(tag).is_dir():
+        if not args.remove_dir:
+            print('Using cached {}'.format(tag))
+            return 0
+        shutil.rmtree(tag)
+    Path(tag).mkdir()
+    bin_path = 'bin/bitcoin-core-{}'.format(tag[1:])
+    match = re.compile('v(.*)(rc[0-9]+)$').search(tag)
+    if match:
+        bin_path = 'bin/bitcoin-core-{}/test.{}'.format(
+            match.group(1), match.group(2))
+    tarball = 'bitcoin-{tag}-{platform}.tar.gz'.format(
+        tag=tag[1:], platform=args.platform)
+    sha256Sums = "SHA256SUMS-{tag}.asc".format(tag=tag[1:])
+    tarballUrl = 'https://bitcoincore.org/{bin_path}/{tarball}'.format(
+        bin_path=bin_path, tarball=tarball)
+    sha256SumsUrl = 'https://bitcoincore.org/{bin_path}/SHA256SUMS.asc'.format(
+        bin_path=bin_path)
+
+    print('Fetching: {tarballUrl}'.format(tarballUrl=tarballUrl))
+    print('Fetching: {sha256SumsUrl}'.format(sha256SumsUrl=sha256SumsUrl))
+
+    header, status = subprocess.Popen(
+        ['curl', '-I', tarballUrl], stdout=subprocess.PIPE).communicate()
+    if re.search("404 Not Found", header.decode("utf-8")):
+        print("Binary tag was not found")
+        return 1
+
+    curlCmds = [
+        ['curl', '-O', tarballUrl],
+        ['curl', "-o", sha256Sums, sha256SumsUrl],
+    ]
+
+    for cmd in curlCmds:
+        ret = subprocess.run(cmd).returncode
+        if ret:
+            return ret
+
+    hasher = hashlib.sha256()
+    with open(tarball, "rb") as afile:
+        buf = afile.read()
+        hasher.update(buf)
+    afile.close()
+    tarballHash = hasher.hexdigest()
+    file = open(sha256Sums, 'r', encoding="utf-8")
+    lst = list(file.readlines())
+    file.close()
+    lastline = lst[len(lst)-1]
+
+    for line in lst:
+        if re.search(tarballHash, line):
+            print("Checksum matched")
+            break
+        elif lastline == line:
+            print("Checksum did not match")
+            Path(tarball).unlink()
+            return 1
+
+    # Bitcoin Core Release Signing Keys v0.11.0+
+    signingKey = "01EA5486DE18A882D4C2684590C8019E36C2E964"
+
+    isKeyPresent = subprocess.run(
+        ["gpg", "--list-keys", signingKey]).returncode
+    if isKeyPresent:
+        return isKeyPresent
+
+    isVerified = subprocess.run(
+        ["gpg", "--verify", sha256Sums]).returncode
+    if isVerified:
+        return isVerified
+
+    # Extract tarball
+    ret = subprocess.run(['tar', '-zxf', tarball, '-C', tag,
+                          '--strip-components=1',
+                          'bitcoin-{tag}'.format(tag=tag[1:])]).returncode
+    if ret:
+        return ret
+
+    Path(tarball).unlink()
+    Path(sha256Sums).unlink()
+    return 0
+
+
+def build_release(tag, args) -> int:
+    githubUrl = "https://github.com/bitcoin/bitcoin"
+    if args.remove_dir:
+        if Path(tag).is_dir():
+            shutil.rmtree(tag)
+    if not Path(tag).is_dir():
+        # fetch new tags
+        subprocess.run(
+            ["git", "fetch", githubUrl, "--tags"])
+        output = subprocess.check_output(['git', 'tag', '-l', tag])
+        if not output:
+            print('Tag {} not found'.format(tag))
+            return 1
+    ret = subprocess.run([
+        'git', 'clone', githubUrl, tag
+    ]).returncode
+    if ret:
+        return ret
+    with pushd(tag):
+        ret = subprocess.run(['git', 'checkout', tag]).returncode
+        if ret:
+            return ret
+        host = args.host
+        if args.depends:
+            with pushd('depends'):
+                ret = subprocess.run(['make', 'NO_QT=1']).returncode
+                if ret:
+                    return ret
+                host = os.environ.get(
+                    'HOST', subprocess.check_output(['./config.guess']))
+        config_flags = '--prefix={pwd}/depends/{host} '.format(
+            pwd=os.getcwd(),
+            host=host) + args.config_flags
+        cmds = [
+            './autogen.sh',
+            './configure {}'.format(config_flags),
+            'make',
+        ]
+        for cmd in cmds:
+            ret = subprocess.run(cmd.split()).returncode
+            if ret:
+                return ret
+        # Move binaries, so they're in the same place as in the
+        # release download
+        Path('bin').mkdir(exist_ok=True)
+        files = ['bitcoind', 'bitcoin-cli', 'bitcoin-tx']
+        for f in files:
+            Path('src/'+f).rename('bin/'+f)
+    return 0
+
+
+def check_host(args) -> int:
+    args.host = os.environ.get('HOST', subprocess.check_output(
+        './depends/config.guess').decode())
+    if args.download_binary:
+        platforms = {
+            'x86_64-*-linux*': 'x86_64-linux-gnu',
+            'x86_64-apple-darwin*': 'osx64',
+        }
+        args.platform = ''
+        for pattern, target in platforms.items():
+            if fnmatch(args.host, pattern):
+                args.platform = target
+        if not args.platform:
+            print('Not sure which binary to download for {}'.format(args.host))
+            return 1
+    return 0
+
+
+def main(args) -> int:
+    if not Path(args.target_dir).is_dir():
+        Path(args.target_dir).mkdir(exist_ok=True, parents=True)
+    print("Releases directory: {}".format(args.target_dir))
+    ret = check_host(args)
+    if ret:
+        return ret
+    if args.download_binary:
+        with pushd(args.target_dir):
+            for tag in args.tags:
+                ret = download_binary(tag, args)
+                if ret:
+                    return ret
+        return 0
+    args.config_flags = os.environ.get('CONFIG_FLAGS', '')
+    args.config_flags += ' --without-gui --disable-tests --disable-bench'
+    with pushd(args.target_dir):
+        for tag in args.tags:
+            ret = build_release(tag, args)
+            if ret:
+                return ret
+    return 0
+
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser(
+        formatter_class=argparse.ArgumentDefaultsHelpFormatter)
+    parser.add_argument('-r', '--remove-dir', action='store_true',
+                        help='remove existing directory.')
+    parser.add_argument('-d', '--depends', action='store_true',
+                        help='use depends.')
+    parser.add_argument('-b', '--download-binary', action='store_true',
+                        help='download release binary.')
+    parser.add_argument('-t', '--target-dir', action='store',
+                        help='target directory.', default='releases')
+    parser.add_argument('tags', nargs='+',
+                        help="release tags. e.g.: v0.18.1 v0.20.0rc2")
+    args = parser.parse_args()
+    sys.exit(main(args))

contrib/devtools/previous_release.sh

@@ -1,152 +0,0 @@
-#!/usr/bin/env bash
-#
-# Copyright (c) 2018-2020 The Bitcoin Core developers
-# Distributed under the MIT software license, see the accompanying
-# file COPYING or http://www.opensource.org/licenses/mit-license.php.
-#
-# Build previous releases.
-
-export LC_ALL=C
-
-CONFIG_FLAGS=""
-FUNCTIONAL_TESTS=0
-DELETE_EXISTING=0
-USE_DEPENDS=0
-DOWNLOAD_BINARY=0
-CONFIG_FLAGS=""
-TARGET="releases"
-
-while getopts ":hfrdbt:" opt; do
-  case $opt in
-    h)
-      echo "Usage: .previous_release.sh [options] tag1 tag2"
-      echo "  options:"
-      echo "  -h   Print this message"
-      echo "  -f   Configure for functional tests"
-      echo "  -r   Remove existing directory"
-      echo "  -d   Use depends"
-      echo "  -b   Download release binary"
-      echo "  -t   Target directory (default: releases)"
-      exit 0
-      ;;
-    f)
-      FUNCTIONAL_TESTS=1
-      CONFIG_FLAGS="$CONFIG_FLAGS --without-gui --disable-tests --disable-bench"
-      ;;
-    r)
-      DELETE_EXISTING=1
-      ;;
-    d)
-      USE_DEPENDS=1
-      ;;
-    b)
-      DOWNLOAD_BINARY=1
-      ;;
-    t)
-      TARGET=$OPTARG
-      ;;
-    \?)
-      echo "Invalid option: -$OPTARG" >&2
-      exit 1
-      ;;
-  esac
-done
-
-shift $((OPTIND-1))
-
-if [ -z "$1" ]; then
-  echo "Specify release tag(s), e.g.: .previous_release v0.15.1"
-  exit 1
-fi
-
-if [ ! -d "$TARGET" ]; then
-  mkdir -p $TARGET
-fi
-
-if [ "$DOWNLOAD_BINARY" -eq "1" ]; then
-  HOST="${HOST:-$(./depends/config.guess)}"
-  case "$HOST" in
-    x86_64-*-linux*)
-      PLATFORM=x86_64-linux-gnu
-      ;;
-    x86_64-apple-darwin*)
-      PLATFORM=osx64
-      ;;
-    *)
-      echo "Not sure which binary to download for $HOST."
-      exit 1
-      ;;
-  esac
-fi
-
-echo "Releases directory: $TARGET"
-pushd "$TARGET" || exit 1
-{
-  for tag in "$@"
-  do
-    if [ "$DELETE_EXISTING" -eq "1" ]; then
-      if [ -d "$tag" ]; then
-        rm -r "$tag"
-      fi
-    fi
-
-    if [ "$DOWNLOAD_BINARY" -eq "0" ]; then
-
-      if [ ! -d "$tag" ]; then
-        if [ -z $(git tag -l "$tag") ]; then
-          echo "Tag $tag not found"
-          exit 1
-        fi
-
-        git clone https://github.com/bitcoin/bitcoin "$tag"
-        pushd "$tag" || exit 1
-        {
-          git checkout "$tag"
-          if [ "$USE_DEPENDS" -eq "1" ]; then
-            pushd depends || exit 1
-            {
-              if [ "$FUNCTIONAL_TESTS" -eq "1" ]; then
-                make NO_QT=1
-              else
-                make
-              fi
-              HOST="${HOST:-$(./config.guess)}"
-            }
-            popd || exit 1
-            CONFIG_FLAGS="--prefix=$PWD/depends/$HOST $CONFIG_FLAGS"
-          fi
-          ./autogen.sh
-          ./configure $CONFIG_FLAGS
-          make
-          # Move binaries, so they're in the same place as in the release download:
-          mkdir bin
-          mv src/bitcoind src/bitcoin-cli src/bitcoin-tx bin
-          if [ "$FUNCTIONAL_TESTS" -eq "0" ]; then
-            mv src/qt/bitcoin-qt bin
-          fi
-        }
-        popd || exit 1
-      fi
-    else
-      if [ -d "$tag" ]; then
-        echo "Using cached $tag"
-      else
-        mkdir "$tag"
-        if [[ "$tag" =~ v(.*)(rc[0-9]+)$ ]]; then
-            BIN_PATH="bin/bitcoin-core-${BASH_REMATCH[1]}/test.${BASH_REMATCH[2]}"
-        else
-            BIN_PATH="bin/bitcoin-core-${tag:1}"
-        fi
-        URL="https://bitcoincore.org/$BIN_PATH/bitcoin-${tag:1}-$PLATFORM.tar.gz"
-        echo "Fetching: $URL"
-        if ! curl -O -f $URL; then
-            echo "Download failed."
-            exit 1
-        fi
-        tar -zxf "bitcoin-${tag:1}-$PLATFORM.tar.gz" -C "$tag" --strip-components=1 "bitcoin-${tag:1}"
-        rm "bitcoin-${tag:1}-$PLATFORM.tar.gz"
-      fi
-    fi
-  done
-}
-popd || exit 1

test/functional/feature_backwards_compatibility.py

@@ -6,7 +6,7 @@
 
 Test various backwards compatibility scenarios. Download the previous node binaries:
 
-contrib/devtools/previous_release.sh -b v0.19.1 v0.18.1 v0.17.1 v0.16.3 v0.15.2
+contrib/devtools/previous_release.py -b v0.19.1 v0.18.1 v0.17.1 v0.16.3 v0.15.2
 
 v0.15.2 is not required by this test, but it is used in wallet_upgradewallet.py.
 Due to a hardfork in regtest, it can't be used to sync nodes.

test/functional/mempool_compatibility.py

@@ -8,7 +8,7 @@ NOTE: The test is designed to prevent cases when compatibility is broken acciden
 In case we need to break mempool compatibility we can continue to use the test by just bumping the version number.
 
 Download node binaries:
-contrib/devtools/previous_release.sh -b v0.19.1 v0.18.1 v0.17.1 v0.16.3 v0.15.2
+contrib/devtools/previous_release.py -b v0.19.1 v0.18.1 v0.17.1 v0.16.3 v0.15.2
 
 Only v0.15.2 is required by this test. The rest is used in other backwards compatibility tests.
 """

test/functional/wallet_upgradewallet.py

@@ -6,7 +6,7 @@
 
 Test upgradewallet RPC. Download node binaries:
 
-contrib/devtools/previous_release.sh -b v0.19.1 v0.18.1 v0.17.1 v0.16.3 v0.15.2
+contrib/devtools/previous_release.py -b v0.19.1 v0.18.1 v0.17.1 v0.16.3 v0.15.2
 
 Only v0.15.2 and v0.16.3 are required by this test. The others are used in feature_backwards_compatibility.py
 """