FreeBSD and dpkg package repository creation and scripts (#84)
Add scripts that create package repositories (as upload-hook) Both dpkg based and FreeBSD based ones are supported. Addresses #73 and #65 Co-authored-by: Hannes Mehnert <hannes@mehnert.org> Co-authored-by: Reynir Björnsson <reynir@reynir.dk> Reviewed-on: https://git.robur.io/robur/builder-web/pulls/84 Co-authored-by: hannes <hannes@mehnert.org> Co-committed-by: hannes <hannes@mehnert.org>
This commit is contained in:
parent
485515e47a
commit
d5f4dc8732
3 changed files with 319 additions and 0 deletions
119
packaging/FreeBSD-repo.sh
Executable file
119
packaging/FreeBSD-repo.sh
Executable file
|
|
@ -0,0 +1,119 @@
|
||||||
|
#!/bin/sh
|
||||||
|
|
||||||
|
set -ex
|
||||||
|
|
||||||
|
prog_NAME=$(basename "${0}")
|
||||||
|
|
||||||
|
warn()
|
||||||
|
{
|
||||||
|
echo "${prog_NAME}: WARN: $*"
|
||||||
|
}
|
||||||
|
|
||||||
|
err()
|
||||||
|
{
|
||||||
|
echo "${prog_NAME}: ERROR: $*" 1>&2
|
||||||
|
}
|
||||||
|
|
||||||
|
die()
|
||||||
|
{
|
||||||
|
echo "${prog_NAME}: ERROR: $*" 1>&2
|
||||||
|
exit 1
|
||||||
|
}
|
||||||
|
|
||||||
|
usage()
|
||||||
|
{
|
||||||
|
cat <<EOM 1>&2
|
||||||
|
usage: ${prog_NAME} [ OPTIONS ] FILE
|
||||||
|
Updates a FreeBSD package repository
|
||||||
|
Options:
|
||||||
|
--build-time=STRING
|
||||||
|
Build timestamp (used for the version of the package).
|
||||||
|
--sha256=STRING
|
||||||
|
Base64 encoded SHA256 digest of the main binary.
|
||||||
|
--job=STRING
|
||||||
|
Job name that was built.
|
||||||
|
EOM
|
||||||
|
exit 1
|
||||||
|
}
|
||||||
|
|
||||||
|
BUILD_TIME=
|
||||||
|
SHA=
|
||||||
|
JOB=
|
||||||
|
|
||||||
|
while [ $# -gt 1 ]; do
|
||||||
|
OPT="$1"
|
||||||
|
|
||||||
|
case "${OPT}" in
|
||||||
|
--build-time=*)
|
||||||
|
BUILD_TIME="${OPT##*=}"
|
||||||
|
;;
|
||||||
|
--sha256=*)
|
||||||
|
SHA="${OPT##*=}"
|
||||||
|
;;
|
||||||
|
--job=*)
|
||||||
|
JOB="${OPT##*=}"
|
||||||
|
;;
|
||||||
|
--*)
|
||||||
|
warn "Ignoring unknown option: '${OPT}'"
|
||||||
|
;;
|
||||||
|
*)
|
||||||
|
err "Unknown option: '${OPT}'"
|
||||||
|
usage
|
||||||
|
;;
|
||||||
|
esac
|
||||||
|
shift
|
||||||
|
done
|
||||||
|
|
||||||
|
[ -z "${BUILD_TIME}" ] && die "The --build-time option must be specified"
|
||||||
|
[ -z "${SHA}" ] && die "The --sha256 option must be specified"
|
||||||
|
[ -z "${JOB}" ] && die "The --job option must be specified"
|
||||||
|
|
||||||
|
FILENAME="${1}"
|
||||||
|
|
||||||
|
: "${REPO:="/usr/local/www/pkg"}"
|
||||||
|
: "${REPO_KEY:="/usr/local/etc/builder-web/repo.key"}"
|
||||||
|
|
||||||
|
if [ "$(basename "${FILENAME}" .pkg)" = "$(basename "${FILENAME}")" ]; then
|
||||||
|
echo "Not a FreeBSD package"
|
||||||
|
exit 0
|
||||||
|
fi
|
||||||
|
|
||||||
|
if ls "${REPO}"/*/All/"${JOB}"-*-"${SHA}".pkg > /dev/null; then
|
||||||
|
echo "Same hash already present, nothing to do"
|
||||||
|
exit 0
|
||||||
|
fi
|
||||||
|
|
||||||
|
TMP=$(mktemp -d -t repak)
|
||||||
|
MANIFEST="${TMP}/+MANIFEST"
|
||||||
|
TMPMANIFEST="${MANIFEST}.tmp"
|
||||||
|
|
||||||
|
cleanup () {
|
||||||
|
rm -rf "${TMP}"
|
||||||
|
}
|
||||||
|
|
||||||
|
trap cleanup EXIT
|
||||||
|
|
||||||
|
PKG_ROOT="${TMP}/pkg"
|
||||||
|
|
||||||
|
tar x -C "${TMP}" -f "${FILENAME}"
|
||||||
|
mkdir "${PKG_ROOT}"
|
||||||
|
mv "${TMP}/usr" "${PKG_ROOT}"
|
||||||
|
|
||||||
|
VERSION=$(jq -r '.version' "${MANIFEST}")
|
||||||
|
NAME=$(jq -r '.name' "${MANIFEST}")
|
||||||
|
FULL_VERSION="${VERSION}-${BUILD_TIME}-${SHA}"
|
||||||
|
|
||||||
|
jq -ca ".version=\"$FULL_VERSION\"" "${MANIFEST}" > "${TMPMANIFEST}"
|
||||||
|
mv "${TMPMANIFEST}" "${MANIFEST}"
|
||||||
|
|
||||||
|
ABI=$(jq -r '.abi' "${MANIFEST}")
|
||||||
|
REPO_DIR="${REPO}/${ABI}"
|
||||||
|
PKG_DIR="${REPO_DIR}/All"
|
||||||
|
|
||||||
|
# to avoid races, first create the package in temporary directory
|
||||||
|
# and then move it before recreating the index
|
||||||
|
pkg create -r "${PKG_ROOT}" -m "${MANIFEST}" -o "${TMP}"
|
||||||
|
mkdir -p "${PKG_DIR}"
|
||||||
|
mv "${TMP}/${NAME}-${FULL_VERSION}.pkg" "${PKG_DIR}"
|
||||||
|
|
||||||
|
pkg repo "${REPO_DIR}" "${REPO_KEY}"
|
||||||
74
packaging/README.md
Normal file
74
packaging/README.md
Normal file
|
|
@ -0,0 +1,74 @@
|
||||||
|
# Package repository creation and update
|
||||||
|
|
||||||
|
Builder-web calls hooks when an upload of a successful build finished. These
|
||||||
|
shell scripts automatically push builds to deb repositories (using aptly) and
|
||||||
|
FreeBSD package repositories (using pkg).
|
||||||
|
|
||||||
|
Thus, as a client of the infrastructure, system packages can be easily
|
||||||
|
installed using the package repositories (and updates are straightforward).
|
||||||
|
|
||||||
|
The tricky part is verioning: different input may result in the same output
|
||||||
|
(i.e. if the build system is updated, it is unlikely this will result in change
|
||||||
|
of output, and clients do not need to update their packages), and also due to
|
||||||
|
the nature of opam, if a dependency (opam package) is released, the output may
|
||||||
|
differ (although the final package version is not increased). We solve the
|
||||||
|
latter by adapting the version number of packages: package version 1.5.2 becomes
|
||||||
|
1.5.2-TIMESTAMP-SHA256. The timestamp is of the form YYYYMMDDhhmmss. The SHA256
|
||||||
|
is the hex-encoded SHA256 checksum of the original binary package and can be
|
||||||
|
used for lookup in the database.
|
||||||
|
|
||||||
|
## DPKG package repository
|
||||||
|
|
||||||
|
The dependencies are aptly and dpkg.
|
||||||
|
|
||||||
|
For the initial setup, a GPG private key is needed:
|
||||||
|
```
|
||||||
|
$ gpg --full-generate-key
|
||||||
|
$ gpg --export --armor > gpg.pub
|
||||||
|
```
|
||||||
|
|
||||||
|
Set REPO_KEYID in the shell script to the key identifier generated
|
||||||
|
(`gpg --list-keys`), and make the gpg.pub available to clients
|
||||||
|
(`cp gpg.pub ~/.aptly/public/`).
|
||||||
|
|
||||||
|
On clients, when the `~/.aptly/public` is served via http(s), add it to your
|
||||||
|
/etc/apt/source.list and import the gpg public key (`apt-key add <gpg.pub>`):
|
||||||
|
|
||||||
|
```
|
||||||
|
deb https://apt.robur.coop/ debian-10 main
|
||||||
|
```
|
||||||
|
|
||||||
|
The `debian-10` can be exchanged with any platform you're building debian
|
||||||
|
packages for.
|
||||||
|
|
||||||
|
## FreeBSD package repository
|
||||||
|
|
||||||
|
The dependency is FreeBSD's pkg utility.
|
||||||
|
|
||||||
|
For the initial setup, a RSA private key is needed:
|
||||||
|
```
|
||||||
|
$ openssl genrsa -out repo.key 4096
|
||||||
|
$ chmod 0400 repo.key
|
||||||
|
$ openssl rsa -in repo.key -out repo.pub -pubout
|
||||||
|
```
|
||||||
|
|
||||||
|
And a directory that acts as package repository (`mkdir /usr/local/www/pkg`).
|
||||||
|
Copy the public key to the package repository
|
||||||
|
(`cp repo.pub /usr/local/www/pkg`) to make it available for clients.
|
||||||
|
|
||||||
|
Both can be configured in the shell script itself (REPO and REPO_KEY). The
|
||||||
|
public key needs to be distributed to clients - e.g. put it at the root of the
|
||||||
|
repository.
|
||||||
|
|
||||||
|
On clients, when that directory is served via http(s), it can be added to
|
||||||
|
/usr/local/etc/pkg/repos/robur.conf:
|
||||||
|
|
||||||
|
```
|
||||||
|
robur: {
|
||||||
|
url: "https://pkg.robur.coop/${ABI}",
|
||||||
|
mirror_type: "srv",
|
||||||
|
signature_type: "pubkey",
|
||||||
|
pubkey: "/path/to/repo.pub",
|
||||||
|
enabled: yes
|
||||||
|
}
|
||||||
|
```
|
||||||
126
packaging/dpkg-repo.sh
Executable file
126
packaging/dpkg-repo.sh
Executable file
|
|
@ -0,0 +1,126 @@
|
||||||
|
#!/bin/sh
|
||||||
|
|
||||||
|
set -ex
|
||||||
|
|
||||||
|
prog_NAME=$(basename "${0}")
|
||||||
|
|
||||||
|
warn()
|
||||||
|
{
|
||||||
|
echo "${prog_NAME}: WARN: $*"
|
||||||
|
}
|
||||||
|
|
||||||
|
err()
|
||||||
|
{
|
||||||
|
echo "${prog_NAME}: ERROR: $*" 1>&2
|
||||||
|
}
|
||||||
|
|
||||||
|
die()
|
||||||
|
{
|
||||||
|
echo "${prog_NAME}: ERROR: $*" 1>&2
|
||||||
|
exit 1
|
||||||
|
}
|
||||||
|
|
||||||
|
usage()
|
||||||
|
{
|
||||||
|
cat <<EOM 1>&2
|
||||||
|
usage: ${prog_NAME} [ OPTIONS ] FILE
|
||||||
|
Updates an aptly package repository
|
||||||
|
Options:
|
||||||
|
--build-time=STRING
|
||||||
|
Build timestamp (used for the version of the package).
|
||||||
|
--sha256=STRING
|
||||||
|
Base64 encoded SHA256 digest of the main binary.
|
||||||
|
--job=STRING
|
||||||
|
Job name that was built.
|
||||||
|
--platform=STRING
|
||||||
|
Platform name on which the build was performed.
|
||||||
|
EOM
|
||||||
|
exit 1
|
||||||
|
}
|
||||||
|
|
||||||
|
BUILD_TIME=
|
||||||
|
SHA=
|
||||||
|
JOB=
|
||||||
|
PLATFORM=
|
||||||
|
|
||||||
|
while [ $# -gt 1 ]; do
|
||||||
|
OPT="$1"
|
||||||
|
|
||||||
|
case "${OPT}" in
|
||||||
|
--build-time=*)
|
||||||
|
BUILD_TIME="${OPT##*=}"
|
||||||
|
;;
|
||||||
|
--sha256=*)
|
||||||
|
SHA="${OPT##*=}"
|
||||||
|
;;
|
||||||
|
--job=*)
|
||||||
|
JOB="${OPT##*=}"
|
||||||
|
;;
|
||||||
|
--platform=*)
|
||||||
|
PLATFORM="${OPT##*=}"
|
||||||
|
;;
|
||||||
|
--*)
|
||||||
|
warn "Ignoring unknown option: '${OPT}'"
|
||||||
|
;;
|
||||||
|
*)
|
||||||
|
err "Unknown option: '${OPT}'"
|
||||||
|
usage
|
||||||
|
;;
|
||||||
|
esac
|
||||||
|
shift
|
||||||
|
done
|
||||||
|
|
||||||
|
[ -z "${BUILD_TIME}" ] && die "The --build-time option must be specified"
|
||||||
|
[ -z "${SHA}" ] && die "The --sha256 option must be specified"
|
||||||
|
[ -z "${JOB}" ] && die "The --job option must be specified"
|
||||||
|
[ -z "${PLATFORM}" ] && die "The --platform option must be specified"
|
||||||
|
|
||||||
|
FILENAME="${1}"
|
||||||
|
|
||||||
|
if [ $(basename "${FILENAME}" .deb) = $(basename "${FILENAME}") ]; then
|
||||||
|
echo "Not a Debian package"
|
||||||
|
exit 0
|
||||||
|
fi
|
||||||
|
|
||||||
|
if aptly repo show -with-packages "${PLATFORM}" | grep "${SHA}" > /dev/null; then
|
||||||
|
echo "Package with same SHA256 already in repository"
|
||||||
|
exit 0
|
||||||
|
fi
|
||||||
|
|
||||||
|
TMP=$(mktemp -d -t debrep)
|
||||||
|
|
||||||
|
cleanup () {
|
||||||
|
rm -rf "${TMP}"
|
||||||
|
}
|
||||||
|
|
||||||
|
trap cleanup EXIT
|
||||||
|
|
||||||
|
dpkg-deb -R "${FILENAME}" "${TMP}"
|
||||||
|
|
||||||
|
VERSION=$(dpkg-deb -f "${FILENAME}" Version)
|
||||||
|
NEW_VERSION="${VERSION}"-"${BUILD_TIME}"-"${SHA}"
|
||||||
|
|
||||||
|
sed -i "" -e "s/Version:.*/Version: ${NEW_VERSION}/g" "${TMP}/DEBIAN/control"
|
||||||
|
|
||||||
|
dpkg-deb --build "${TMP}" "${TMP}"
|
||||||
|
|
||||||
|
REPO_EXISTS=1
|
||||||
|
if ! aptly repo show "${PLATFORM}" > /dev/null 2>&1; then
|
||||||
|
REPO_EXISTS=0
|
||||||
|
fi
|
||||||
|
|
||||||
|
if [ "${REPO_EXISTS}" -eq 0 ]; then
|
||||||
|
aptly repo create --distribution="${PLATFORM}" "${PLATFORM}"
|
||||||
|
fi
|
||||||
|
|
||||||
|
aptly repo add "${PLATFORM}" "${TMP}"
|
||||||
|
|
||||||
|
: "${REPO_KEYID:="D5E2DC92617877EDF7D4FD4345EA05FB7E26053D"}"
|
||||||
|
|
||||||
|
if [ "${REPO_EXISTS}" -eq 0 ]; then
|
||||||
|
aptly publish repo -gpg-key="${REPO_KEYID}" -architectures=all "${PLATFORM}"
|
||||||
|
else
|
||||||
|
aptly publish update -gpg-key="${REPO_KEYID}" "${PLATFORM}"
|
||||||
|
fi
|
||||||
|
|
||||||
|
# TODO architectures
|
||||||
Loading…
Reference in a new issue