Merge pull request #948 from vector-im/feature/artifacts_downloading

Feature/artifacts downloading
This commit is contained in:
Benoit Marty 2020-02-10 14:52:25 +01:00 committed by GitHub
commit c97d298166
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 218 additions and 0 deletions

1
.gitignore vendored
View File

@ -4,6 +4,7 @@
# idea files: exclude everything except dictionnaries # idea files: exclude everything except dictionnaries
.idea/caches .idea/caches
.idea/libraries .idea/libraries
.idea/inspectionProfiles
.idea/*.xml .idea/*.xml
.DS_Store .DS_Store
/build /build

View File

@ -0,0 +1,164 @@
#!/usr/bin/env python3
#
# Copyright 2020 New Vector Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
import argparse
import hashlib
import json
import os
# Run `pip3 install requests` if not installed yet
import requests
# This script downloads artifacts from buildkite.
# Ref: https://buildkite.com/docs/apis/rest-api/artifacts#download-an-artifact
# Those two variable are specific to the RiotX project
ORG_SLUG = "matrix-dot-org"
PIPELINE_SLUG = "riotx-android"
### Arguments
parser = argparse.ArgumentParser(description='Download artifacts from Buildkite.')
parser.add_argument('-t',
'--token',
required=True,
help='The buildkite token.')
parser.add_argument('-b',
'--build',
type=int,
required=True,
help='the buildkite build number.')
parser.add_argument('-e',
'--expecting',
type=int,
default=-1,
help='the expected number of artifacts. If omitted, no check will be done.')
parser.add_argument('-d',
'--directory',
default="",
help='the target directory, where files will be downloaded. If not provided the build number will be used to create a directory.')
parser.add_argument('-v',
'--verbose',
help="increase output verbosity.",
action="store_true")
parser.add_argument('-s',
'--simulate',
help="simulate action, do not create folder or download any file.",
action="store_true")
args = parser.parse_args()
# parser has checked that the build was an int, convert to String for the rest of the script
build_str = str(args.build)
if args.verbose:
print("Argument:")
print(args)
headers = {'Authorization': "Bearer %s" % args.token}
base_url = "https://api.buildkite.com/v2/organizations/%s/pipelines/%s/builds/%s" % (ORG_SLUG, PIPELINE_SLUG, build_str)
### Fetch build state
buildkite_build_state_url = base_url
print("Getting build state of project %s/%s build %s" % (ORG_SLUG, PIPELINE_SLUG, build_str))
if args.verbose:
print("Url: %s" % buildkite_build_state_url)
r0 = requests.get(buildkite_build_state_url, headers=headers)
data0 = json.loads(r0.content.decode())
if args.verbose:
print("Json data:")
print(data0)
print(" git branch : %s" % data0.get('branch'))
print(" git commit : \"%s\"" % data0.get('commit'))
print(" git commit message : \"%s\"" % data0.get('message'))
print(" build state : %s" % data0.get('state'))
if data0.get('state') != 'passed':
print("❌ Error, the build failed (state: %s)" % data0.get('state'))
exit(0)
### Fetch artifacts list
buildkite_artifacts_url = base_url + "/artifacts"
print("Getting artifacts list of project %s/%s build %s" % (ORG_SLUG, PIPELINE_SLUG, build_str))
if args.verbose:
print("Url: %s" % buildkite_artifacts_url)
r = requests.get(buildkite_artifacts_url, headers=headers)
data = json.loads(r.content.decode())
print(" %d artifact(s) found." % len(data))
if args.expecting != -1 and args.expecting != len(data):
print("Error, expecting %d artifacts and found %d." % (args.expecting, len(data)))
exit(1)
if args.verbose:
print("Json data:")
print(data)
if args.verbose:
print("Create subfolder %s to download artifacts..." % build_str)
if args.directory == "":
targetDir = build_str
else:
targetDir = args.directory
if not args.simulate:
os.mkdir(targetDir)
error = False
for elt in data:
if args.verbose:
print()
print("Artifact info:")
for key, value in elt.items():
print(" %s: %s" % (key, str(value)))
url = elt.get("download_url")
filename = elt.get("filename")
target = targetDir + "/" + filename
print("Downloading %s to '%s'..." % (filename, targetDir))
if not args.simulate:
# open file to write in binary mode
with open(target, "wb") as file:
# get request
response = requests.get(url, headers=headers)
# write to file
file.write(response.content)
print("Verifying checksum...")
# open file to read in binary mode
with open(target, "rb") as file:
data = file.read()
hash = hashlib.sha1(data).hexdigest()
if elt.get("sha1sum") != hash:
error = True
print("❌ Checksum mismatch: expecting %s and get %s" % (elt.get("sha1sum"), hash))
if error:
print("❌ Error(s) occurred, check the log")
exit(1)
else:
print("Done!")

View File

@ -0,0 +1,53 @@
#!/usr/bin/env bash
# Copy and adaptation of ./sign_apk.sh, which takes 2 more params: key store pass and key pass.
# It's unsafe to use it because it takes password as parameter, so passwords will
# remain in the terminal history.
set -e
if [[ -z "${ANDROID_HOME}" ]]; then
echo "Env variable ANDROID_HOME is not set, should be set to something like ~/Library/Android/sdk"
exit 1
fi
if [[ "$#" -ne 4 ]]; then
echo "Usage: $0 KEYSTORE_PATH APK KS_PASS KEY_PASS" >&2
exit 1
fi
# Get the command line parameters
PARAM_KEYSTORE_PATH=$1
PARAM_APK=$2
PARAM_KS_PASS=$3
PARAM_KEY_PASS=$4
# Other params
BUILD_TOOLS_VERSION="28.0.3"
MIN_SDK_VERSION=19
echo "Signing APK with build-tools version ${BUILD_TOOLS_VERSION} for min SDK version ${MIN_SDK_VERSION}..."
APK_SIGNER_PATH=${ANDROID_HOME}/build-tools/${BUILD_TOOLS_VERSION}
${APK_SIGNER_PATH}/apksigner sign \
-v \
--ks ${PARAM_KEYSTORE_PATH} \
--ks-pass pass:${PARAM_KS_PASS} \
--ks-key-alias riot.im \
--key-pass pass:${PARAM_KEY_PASS} \
--min-sdk-version ${MIN_SDK_VERSION} \
${PARAM_APK}
# Verify the signature
echo "Verifying the signature..."
# Note: we ignore warning on META-INF files
${APK_SIGNER_PATH}/apksigner verify \
-v \
--min-sdk-version ${MIN_SDK_VERSION} \
${PARAM_APK} \
| grep -v "WARNING: META-INF/"
echo
echo "Congratulations! The APK ${PARAM_APK} is now signed!"