From d5f4dc87327ddb841b9889d42c62fd1c9cbcfc2e Mon Sep 17 00:00:00 2001 From: hannes Date: Wed, 23 Feb 2022 16:08:26 +0000 Subject: [PATCH] FreeBSD and dpkg package repository creation and scripts (#84) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add scripts that create package repositories (as upload-hook) Both dpkg based and FreeBSD based ones are supported. Addresses #73 and #65 Co-authored-by: Hannes Mehnert Co-authored-by: Reynir Björnsson Reviewed-on: https://git.robur.io/robur/builder-web/pulls/84 Co-authored-by: hannes Co-committed-by: hannes --- packaging/FreeBSD-repo.sh | 119 +++++++++++++++++++++++++++++++++++ packaging/README.md | 74 ++++++++++++++++++++++ packaging/dpkg-repo.sh | 126 ++++++++++++++++++++++++++++++++++++++ 3 files changed, 319 insertions(+) create mode 100755 packaging/FreeBSD-repo.sh create mode 100644 packaging/README.md create mode 100755 packaging/dpkg-repo.sh diff --git a/packaging/FreeBSD-repo.sh b/packaging/FreeBSD-repo.sh new file mode 100755 index 0000000..c40d8b9 --- /dev/null +++ b/packaging/FreeBSD-repo.sh @@ -0,0 +1,119 @@ +#!/bin/sh + +set -ex + +prog_NAME=$(basename "${0}") + +warn() +{ + echo "${prog_NAME}: WARN: $*" +} + +err() +{ + echo "${prog_NAME}: ERROR: $*" 1>&2 +} + +die() +{ + echo "${prog_NAME}: ERROR: $*" 1>&2 + exit 1 +} + +usage() +{ + cat <&2 +usage: ${prog_NAME} [ OPTIONS ] FILE +Updates a FreeBSD package repository +Options: + --build-time=STRING + Build timestamp (used for the version of the package). + --sha256=STRING + Base64 encoded SHA256 digest of the main binary. + --job=STRING + Job name that was built. +EOM + exit 1 +} + +BUILD_TIME= +SHA= +JOB= + +while [ $# -gt 1 ]; do + OPT="$1" + + case "${OPT}" in + --build-time=*) + BUILD_TIME="${OPT##*=}" + ;; + --sha256=*) + SHA="${OPT##*=}" + ;; + --job=*) + JOB="${OPT##*=}" + ;; + --*) + warn "Ignoring unknown option: '${OPT}'" + ;; + *) + err "Unknown option: '${OPT}'" + usage + ;; + esac + shift +done + +[ -z "${BUILD_TIME}" ] && die "The --build-time option must be specified" +[ -z "${SHA}" ] && die "The --sha256 option must be specified" +[ -z "${JOB}" ] && die "The --job option must be specified" + +FILENAME="${1}" + +: "${REPO:="/usr/local/www/pkg"}" +: "${REPO_KEY:="/usr/local/etc/builder-web/repo.key"}" + +if [ "$(basename "${FILENAME}" .pkg)" = "$(basename "${FILENAME}")" ]; then + echo "Not a FreeBSD package" + exit 0 +fi + +if ls "${REPO}"/*/All/"${JOB}"-*-"${SHA}".pkg > /dev/null; then + echo "Same hash already present, nothing to do" + exit 0 +fi + +TMP=$(mktemp -d -t repak) +MANIFEST="${TMP}/+MANIFEST" +TMPMANIFEST="${MANIFEST}.tmp" + +cleanup () { + rm -rf "${TMP}" +} + +trap cleanup EXIT + +PKG_ROOT="${TMP}/pkg" + +tar x -C "${TMP}" -f "${FILENAME}" +mkdir "${PKG_ROOT}" +mv "${TMP}/usr" "${PKG_ROOT}" + +VERSION=$(jq -r '.version' "${MANIFEST}") +NAME=$(jq -r '.name' "${MANIFEST}") +FULL_VERSION="${VERSION}-${BUILD_TIME}-${SHA}" + +jq -ca ".version=\"$FULL_VERSION\"" "${MANIFEST}" > "${TMPMANIFEST}" +mv "${TMPMANIFEST}" "${MANIFEST}" + +ABI=$(jq -r '.abi' "${MANIFEST}") +REPO_DIR="${REPO}/${ABI}" +PKG_DIR="${REPO_DIR}/All" + +# to avoid races, first create the package in temporary directory +# and then move it before recreating the index +pkg create -r "${PKG_ROOT}" -m "${MANIFEST}" -o "${TMP}" +mkdir -p "${PKG_DIR}" +mv "${TMP}/${NAME}-${FULL_VERSION}.pkg" "${PKG_DIR}" + +pkg repo "${REPO_DIR}" "${REPO_KEY}" diff --git a/packaging/README.md b/packaging/README.md new file mode 100644 index 0000000..2c73804 --- /dev/null +++ b/packaging/README.md @@ -0,0 +1,74 @@ +# Package repository creation and update + +Builder-web calls hooks when an upload of a successful build finished. These +shell scripts automatically push builds to deb repositories (using aptly) and +FreeBSD package repositories (using pkg). + +Thus, as a client of the infrastructure, system packages can be easily +installed using the package repositories (and updates are straightforward). + +The tricky part is verioning: different input may result in the same output +(i.e. if the build system is updated, it is unlikely this will result in change +of output, and clients do not need to update their packages), and also due to +the nature of opam, if a dependency (opam package) is released, the output may +differ (although the final package version is not increased). We solve the +latter by adapting the version number of packages: package version 1.5.2 becomes +1.5.2-TIMESTAMP-SHA256. The timestamp is of the form YYYYMMDDhhmmss. The SHA256 +is the hex-encoded SHA256 checksum of the original binary package and can be +used for lookup in the database. + +## DPKG package repository + +The dependencies are aptly and dpkg. + +For the initial setup, a GPG private key is needed: +``` +$ gpg --full-generate-key +$ gpg --export --armor > gpg.pub +``` + +Set REPO_KEYID in the shell script to the key identifier generated +(`gpg --list-keys`), and make the gpg.pub available to clients +(`cp gpg.pub ~/.aptly/public/`). + +On clients, when the `~/.aptly/public` is served via http(s), add it to your +/etc/apt/source.list and import the gpg public key (`apt-key add `): + +``` +deb https://apt.robur.coop/ debian-10 main +``` + +The `debian-10` can be exchanged with any platform you're building debian +packages for. + +## FreeBSD package repository + +The dependency is FreeBSD's pkg utility. + +For the initial setup, a RSA private key is needed: +``` +$ openssl genrsa -out repo.key 4096 +$ chmod 0400 repo.key +$ openssl rsa -in repo.key -out repo.pub -pubout +``` + +And a directory that acts as package repository (`mkdir /usr/local/www/pkg`). +Copy the public key to the package repository +(`cp repo.pub /usr/local/www/pkg`) to make it available for clients. + +Both can be configured in the shell script itself (REPO and REPO_KEY). The +public key needs to be distributed to clients - e.g. put it at the root of the +repository. + +On clients, when that directory is served via http(s), it can be added to +/usr/local/etc/pkg/repos/robur.conf: + +``` +robur: { + url: "https://pkg.robur.coop/${ABI}", + mirror_type: "srv", + signature_type: "pubkey", + pubkey: "/path/to/repo.pub", + enabled: yes +} +``` diff --git a/packaging/dpkg-repo.sh b/packaging/dpkg-repo.sh new file mode 100755 index 0000000..0d6747e --- /dev/null +++ b/packaging/dpkg-repo.sh @@ -0,0 +1,126 @@ +#!/bin/sh + +set -ex + +prog_NAME=$(basename "${0}") + +warn() +{ + echo "${prog_NAME}: WARN: $*" +} + +err() +{ + echo "${prog_NAME}: ERROR: $*" 1>&2 +} + +die() +{ + echo "${prog_NAME}: ERROR: $*" 1>&2 + exit 1 +} + +usage() +{ + cat <&2 +usage: ${prog_NAME} [ OPTIONS ] FILE +Updates an aptly package repository +Options: + --build-time=STRING + Build timestamp (used for the version of the package). + --sha256=STRING + Base64 encoded SHA256 digest of the main binary. + --job=STRING + Job name that was built. + --platform=STRING + Platform name on which the build was performed. +EOM + exit 1 +} + +BUILD_TIME= +SHA= +JOB= +PLATFORM= + +while [ $# -gt 1 ]; do + OPT="$1" + + case "${OPT}" in + --build-time=*) + BUILD_TIME="${OPT##*=}" + ;; + --sha256=*) + SHA="${OPT##*=}" + ;; + --job=*) + JOB="${OPT##*=}" + ;; + --platform=*) + PLATFORM="${OPT##*=}" + ;; + --*) + warn "Ignoring unknown option: '${OPT}'" + ;; + *) + err "Unknown option: '${OPT}'" + usage + ;; + esac + shift +done + +[ -z "${BUILD_TIME}" ] && die "The --build-time option must be specified" +[ -z "${SHA}" ] && die "The --sha256 option must be specified" +[ -z "${JOB}" ] && die "The --job option must be specified" +[ -z "${PLATFORM}" ] && die "The --platform option must be specified" + +FILENAME="${1}" + +if [ $(basename "${FILENAME}" .deb) = $(basename "${FILENAME}") ]; then + echo "Not a Debian package" + exit 0 +fi + +if aptly repo show -with-packages "${PLATFORM}" | grep "${SHA}" > /dev/null; then + echo "Package with same SHA256 already in repository" + exit 0 +fi + +TMP=$(mktemp -d -t debrep) + +cleanup () { + rm -rf "${TMP}" +} + +trap cleanup EXIT + +dpkg-deb -R "${FILENAME}" "${TMP}" + +VERSION=$(dpkg-deb -f "${FILENAME}" Version) +NEW_VERSION="${VERSION}"-"${BUILD_TIME}"-"${SHA}" + +sed -i "" -e "s/Version:.*/Version: ${NEW_VERSION}/g" "${TMP}/DEBIAN/control" + +dpkg-deb --build "${TMP}" "${TMP}" + +REPO_EXISTS=1 +if ! aptly repo show "${PLATFORM}" > /dev/null 2>&1; then + REPO_EXISTS=0 +fi + +if [ "${REPO_EXISTS}" -eq 0 ]; then + aptly repo create --distribution="${PLATFORM}" "${PLATFORM}" +fi + +aptly repo add "${PLATFORM}" "${TMP}" + +: "${REPO_KEYID:="D5E2DC92617877EDF7D4FD4345EA05FB7E26053D"}" + +if [ "${REPO_EXISTS}" -eq 0 ]; then + aptly publish repo -gpg-key="${REPO_KEYID}" -architectures=all "${PLATFORM}" +else + aptly publish update -gpg-key="${REPO_KEYID}" "${PLATFORM}" +fi + +# TODO architectures