FreeBSD and dpkg package repository creation and scripts (#84)
Add scripts that create package repositories (as upload-hook) Both dpkg based and FreeBSD based ones are supported. Addresses #73 and #65 Co-authored-by: Hannes Mehnert <hannes@mehnert.org> Co-authored-by: Reynir Björnsson <reynir@reynir.dk> Reviewed-on: https://git.robur.io/robur/builder-web/pulls/84 Co-authored-by: hannes <hannes@mehnert.org> Co-committed-by: hannes <hannes@mehnert.org>
This commit is contained in:
parent
485515e47a
commit
d5f4dc8732
3 changed files with 319 additions and 0 deletions
119
packaging/FreeBSD-repo.sh
Executable file
119
packaging/FreeBSD-repo.sh
Executable file
|
|
@ -0,0 +1,119 @@
|
|||
#!/bin/sh
|
||||
|
||||
set -ex
|
||||
|
||||
prog_NAME=$(basename "${0}")
|
||||
|
||||
warn()
|
||||
{
|
||||
echo "${prog_NAME}: WARN: $*"
|
||||
}
|
||||
|
||||
err()
|
||||
{
|
||||
echo "${prog_NAME}: ERROR: $*" 1>&2
|
||||
}
|
||||
|
||||
die()
|
||||
{
|
||||
echo "${prog_NAME}: ERROR: $*" 1>&2
|
||||
exit 1
|
||||
}
|
||||
|
||||
usage()
|
||||
{
|
||||
cat <<EOM 1>&2
|
||||
usage: ${prog_NAME} [ OPTIONS ] FILE
|
||||
Updates a FreeBSD package repository
|
||||
Options:
|
||||
--build-time=STRING
|
||||
Build timestamp (used for the version of the package).
|
||||
--sha256=STRING
|
||||
Base64 encoded SHA256 digest of the main binary.
|
||||
--job=STRING
|
||||
Job name that was built.
|
||||
EOM
|
||||
exit 1
|
||||
}
|
||||
|
||||
BUILD_TIME=
|
||||
SHA=
|
||||
JOB=
|
||||
|
||||
while [ $# -gt 1 ]; do
|
||||
OPT="$1"
|
||||
|
||||
case "${OPT}" in
|
||||
--build-time=*)
|
||||
BUILD_TIME="${OPT##*=}"
|
||||
;;
|
||||
--sha256=*)
|
||||
SHA="${OPT##*=}"
|
||||
;;
|
||||
--job=*)
|
||||
JOB="${OPT##*=}"
|
||||
;;
|
||||
--*)
|
||||
warn "Ignoring unknown option: '${OPT}'"
|
||||
;;
|
||||
*)
|
||||
err "Unknown option: '${OPT}'"
|
||||
usage
|
||||
;;
|
||||
esac
|
||||
shift
|
||||
done
|
||||
|
||||
[ -z "${BUILD_TIME}" ] && die "The --build-time option must be specified"
|
||||
[ -z "${SHA}" ] && die "The --sha256 option must be specified"
|
||||
[ -z "${JOB}" ] && die "The --job option must be specified"
|
||||
|
||||
FILENAME="${1}"
|
||||
|
||||
: "${REPO:="/usr/local/www/pkg"}"
|
||||
: "${REPO_KEY:="/usr/local/etc/builder-web/repo.key"}"
|
||||
|
||||
if [ "$(basename "${FILENAME}" .pkg)" = "$(basename "${FILENAME}")" ]; then
|
||||
echo "Not a FreeBSD package"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
if ls "${REPO}"/*/All/"${JOB}"-*-"${SHA}".pkg > /dev/null; then
|
||||
echo "Same hash already present, nothing to do"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
TMP=$(mktemp -d -t repak)
|
||||
MANIFEST="${TMP}/+MANIFEST"
|
||||
TMPMANIFEST="${MANIFEST}.tmp"
|
||||
|
||||
cleanup () {
|
||||
rm -rf "${TMP}"
|
||||
}
|
||||
|
||||
trap cleanup EXIT
|
||||
|
||||
PKG_ROOT="${TMP}/pkg"
|
||||
|
||||
tar x -C "${TMP}" -f "${FILENAME}"
|
||||
mkdir "${PKG_ROOT}"
|
||||
mv "${TMP}/usr" "${PKG_ROOT}"
|
||||
|
||||
VERSION=$(jq -r '.version' "${MANIFEST}")
|
||||
NAME=$(jq -r '.name' "${MANIFEST}")
|
||||
FULL_VERSION="${VERSION}-${BUILD_TIME}-${SHA}"
|
||||
|
||||
jq -ca ".version=\"$FULL_VERSION\"" "${MANIFEST}" > "${TMPMANIFEST}"
|
||||
mv "${TMPMANIFEST}" "${MANIFEST}"
|
||||
|
||||
ABI=$(jq -r '.abi' "${MANIFEST}")
|
||||
REPO_DIR="${REPO}/${ABI}"
|
||||
PKG_DIR="${REPO_DIR}/All"
|
||||
|
||||
# to avoid races, first create the package in temporary directory
|
||||
# and then move it before recreating the index
|
||||
pkg create -r "${PKG_ROOT}" -m "${MANIFEST}" -o "${TMP}"
|
||||
mkdir -p "${PKG_DIR}"
|
||||
mv "${TMP}/${NAME}-${FULL_VERSION}.pkg" "${PKG_DIR}"
|
||||
|
||||
pkg repo "${REPO_DIR}" "${REPO_KEY}"
|
||||
74
packaging/README.md
Normal file
74
packaging/README.md
Normal file
|
|
@ -0,0 +1,74 @@
|
|||
# Package repository creation and update
|
||||
|
||||
Builder-web calls hooks when an upload of a successful build finished. These
|
||||
shell scripts automatically push builds to deb repositories (using aptly) and
|
||||
FreeBSD package repositories (using pkg).
|
||||
|
||||
Thus, as a client of the infrastructure, system packages can be easily
|
||||
installed using the package repositories (and updates are straightforward).
|
||||
|
||||
The tricky part is verioning: different input may result in the same output
|
||||
(i.e. if the build system is updated, it is unlikely this will result in change
|
||||
of output, and clients do not need to update their packages), and also due to
|
||||
the nature of opam, if a dependency (opam package) is released, the output may
|
||||
differ (although the final package version is not increased). We solve the
|
||||
latter by adapting the version number of packages: package version 1.5.2 becomes
|
||||
1.5.2-TIMESTAMP-SHA256. The timestamp is of the form YYYYMMDDhhmmss. The SHA256
|
||||
is the hex-encoded SHA256 checksum of the original binary package and can be
|
||||
used for lookup in the database.
|
||||
|
||||
## DPKG package repository
|
||||
|
||||
The dependencies are aptly and dpkg.
|
||||
|
||||
For the initial setup, a GPG private key is needed:
|
||||
```
|
||||
$ gpg --full-generate-key
|
||||
$ gpg --export --armor > gpg.pub
|
||||
```
|
||||
|
||||
Set REPO_KEYID in the shell script to the key identifier generated
|
||||
(`gpg --list-keys`), and make the gpg.pub available to clients
|
||||
(`cp gpg.pub ~/.aptly/public/`).
|
||||
|
||||
On clients, when the `~/.aptly/public` is served via http(s), add it to your
|
||||
/etc/apt/source.list and import the gpg public key (`apt-key add <gpg.pub>`):
|
||||
|
||||
```
|
||||
deb https://apt.robur.coop/ debian-10 main
|
||||
```
|
||||
|
||||
The `debian-10` can be exchanged with any platform you're building debian
|
||||
packages for.
|
||||
|
||||
## FreeBSD package repository
|
||||
|
||||
The dependency is FreeBSD's pkg utility.
|
||||
|
||||
For the initial setup, a RSA private key is needed:
|
||||
```
|
||||
$ openssl genrsa -out repo.key 4096
|
||||
$ chmod 0400 repo.key
|
||||
$ openssl rsa -in repo.key -out repo.pub -pubout
|
||||
```
|
||||
|
||||
And a directory that acts as package repository (`mkdir /usr/local/www/pkg`).
|
||||
Copy the public key to the package repository
|
||||
(`cp repo.pub /usr/local/www/pkg`) to make it available for clients.
|
||||
|
||||
Both can be configured in the shell script itself (REPO and REPO_KEY). The
|
||||
public key needs to be distributed to clients - e.g. put it at the root of the
|
||||
repository.
|
||||
|
||||
On clients, when that directory is served via http(s), it can be added to
|
||||
/usr/local/etc/pkg/repos/robur.conf:
|
||||
|
||||
```
|
||||
robur: {
|
||||
url: "https://pkg.robur.coop/${ABI}",
|
||||
mirror_type: "srv",
|
||||
signature_type: "pubkey",
|
||||
pubkey: "/path/to/repo.pub",
|
||||
enabled: yes
|
||||
}
|
||||
```
|
||||
126
packaging/dpkg-repo.sh
Executable file
126
packaging/dpkg-repo.sh
Executable file
|
|
@ -0,0 +1,126 @@
|
|||
#!/bin/sh
|
||||
|
||||
set -ex
|
||||
|
||||
prog_NAME=$(basename "${0}")
|
||||
|
||||
warn()
|
||||
{
|
||||
echo "${prog_NAME}: WARN: $*"
|
||||
}
|
||||
|
||||
err()
|
||||
{
|
||||
echo "${prog_NAME}: ERROR: $*" 1>&2
|
||||
}
|
||||
|
||||
die()
|
||||
{
|
||||
echo "${prog_NAME}: ERROR: $*" 1>&2
|
||||
exit 1
|
||||
}
|
||||
|
||||
usage()
|
||||
{
|
||||
cat <<EOM 1>&2
|
||||
usage: ${prog_NAME} [ OPTIONS ] FILE
|
||||
Updates an aptly package repository
|
||||
Options:
|
||||
--build-time=STRING
|
||||
Build timestamp (used for the version of the package).
|
||||
--sha256=STRING
|
||||
Base64 encoded SHA256 digest of the main binary.
|
||||
--job=STRING
|
||||
Job name that was built.
|
||||
--platform=STRING
|
||||
Platform name on which the build was performed.
|
||||
EOM
|
||||
exit 1
|
||||
}
|
||||
|
||||
BUILD_TIME=
|
||||
SHA=
|
||||
JOB=
|
||||
PLATFORM=
|
||||
|
||||
while [ $# -gt 1 ]; do
|
||||
OPT="$1"
|
||||
|
||||
case "${OPT}" in
|
||||
--build-time=*)
|
||||
BUILD_TIME="${OPT##*=}"
|
||||
;;
|
||||
--sha256=*)
|
||||
SHA="${OPT##*=}"
|
||||
;;
|
||||
--job=*)
|
||||
JOB="${OPT##*=}"
|
||||
;;
|
||||
--platform=*)
|
||||
PLATFORM="${OPT##*=}"
|
||||
;;
|
||||
--*)
|
||||
warn "Ignoring unknown option: '${OPT}'"
|
||||
;;
|
||||
*)
|
||||
err "Unknown option: '${OPT}'"
|
||||
usage
|
||||
;;
|
||||
esac
|
||||
shift
|
||||
done
|
||||
|
||||
[ -z "${BUILD_TIME}" ] && die "The --build-time option must be specified"
|
||||
[ -z "${SHA}" ] && die "The --sha256 option must be specified"
|
||||
[ -z "${JOB}" ] && die "The --job option must be specified"
|
||||
[ -z "${PLATFORM}" ] && die "The --platform option must be specified"
|
||||
|
||||
FILENAME="${1}"
|
||||
|
||||
if [ $(basename "${FILENAME}" .deb) = $(basename "${FILENAME}") ]; then
|
||||
echo "Not a Debian package"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
if aptly repo show -with-packages "${PLATFORM}" | grep "${SHA}" > /dev/null; then
|
||||
echo "Package with same SHA256 already in repository"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
TMP=$(mktemp -d -t debrep)
|
||||
|
||||
cleanup () {
|
||||
rm -rf "${TMP}"
|
||||
}
|
||||
|
||||
trap cleanup EXIT
|
||||
|
||||
dpkg-deb -R "${FILENAME}" "${TMP}"
|
||||
|
||||
VERSION=$(dpkg-deb -f "${FILENAME}" Version)
|
||||
NEW_VERSION="${VERSION}"-"${BUILD_TIME}"-"${SHA}"
|
||||
|
||||
sed -i "" -e "s/Version:.*/Version: ${NEW_VERSION}/g" "${TMP}/DEBIAN/control"
|
||||
|
||||
dpkg-deb --build "${TMP}" "${TMP}"
|
||||
|
||||
REPO_EXISTS=1
|
||||
if ! aptly repo show "${PLATFORM}" > /dev/null 2>&1; then
|
||||
REPO_EXISTS=0
|
||||
fi
|
||||
|
||||
if [ "${REPO_EXISTS}" -eq 0 ]; then
|
||||
aptly repo create --distribution="${PLATFORM}" "${PLATFORM}"
|
||||
fi
|
||||
|
||||
aptly repo add "${PLATFORM}" "${TMP}"
|
||||
|
||||
: "${REPO_KEYID:="D5E2DC92617877EDF7D4FD4345EA05FB7E26053D"}"
|
||||
|
||||
if [ "${REPO_EXISTS}" -eq 0 ]; then
|
||||
aptly publish repo -gpg-key="${REPO_KEYID}" -architectures=all "${PLATFORM}"
|
||||
else
|
||||
aptly publish update -gpg-key="${REPO_KEYID}" "${PLATFORM}"
|
||||
fi
|
||||
|
||||
# TODO architectures
|
||||
Loading…
Reference in a new issue