Merge pull request 'Visualization script enhancements' (#136) from viz-script-fixes into main

Reviewed-on: https://git.robur.io/robur/builder-web/pulls/136

packaging/batch-viz.sh

@@ -127,8 +127,22 @@ fi
 ATTEMPTED_VIZS=0
 FAILED_VIZS=0
 
-for i in $(find "${DATA_DIR}" -type f -path \*output/bin\*); do
+distinct-input () {
-    UUID=$(echo "${i}" | rev | cut -d '/' -f 4 | rev)
+    {
+        sqlite3 "${DATA_DIR}/builder.sqlite3" "SELECT b.uuid
+            FROM build b
+            JOIN build_artifact opam ON opam.build = b.id
+            WHERE opam.filepath = 'opam-switch' AND b.main_binary NOT NULL
+            GROUP BY opam.sha256;"
+        sqlite3 "${DATA_DIR}/builder.sqlite3" "SELECT b.uuid
+            FROM build b
+            JOIN build_artifact debug ON debug.build = b.id
+            WHERE debug.filepath LIKE '%.debug' AND b.main_binary NOT NULL
+            GROUP BY debug.sha256;"
+    } | sort -u
+}
+
+for UUID in $(distinct-input); do
     if ! "$VISUALIZATIONS_CMD" \
          --data-dir="${DATA_DIR}" \
          --cache-dir="${CACHE_DIR}" \

packaging/visualizations.sh

@@ -70,24 +70,24 @@ done
 [ -z "${CACHE_DIR}" ] && die "The --cache-dir option must be specified"
 [ -z "${DATA_DIR}" ] && die "The --data-dir option must be specified"
 
-info "processing UUID '$UUID'"
+info "processing UUID '${UUID}'"
 
 DB="${DATA_DIR}/builder.sqlite3"
 
 get_main_binary () {
     sqlite3 "${DB}" "SELECT ba.localpath FROM build AS b
         JOIN build_artifact AS ba ON ba.build = b.id AND b.main_binary = ba.id
-        WHERE uuid = '$UUID';"
+        WHERE uuid = '${UUID}';"
 }
 
 BIN="${DATA_DIR}/$(get_main_binary)" || die "Failed to get main binary from database"
-[ -z "${BIN}" ] && die "No main-binary found in db '$DB' for build '$UUID'"
+[ -z "${BIN}" ] && die "No main-binary found in db '${DB}' for build '${UUID}'"
 
 get_debug_binary () {
     sqlite3 "${DB}" "SELECT ba.localpath FROM build AS b
         JOIN build_artifact AS ba ON ba.build = b.id
         WHERE
-          uuid = '$UUID'
+          uuid = '${UUID}'
           AND ba.localpath LIKE '%.debug';"
 }
 
@@ -97,19 +97,19 @@ get_opam_switch () {
     sqlite3 "${DB}" "SELECT ba.localpath FROM build AS b
         JOIN build_artifact AS ba ON ba.build = b.id
         WHERE
-          uuid = '$UUID'
+          uuid = '${UUID}'
           AND ba.filepath = 'opam-switch';"
 }
 
 OPAM_SWITCH="$(get_opam_switch)" || die "Failed to get opam switch from database"
-[ -z "${OPAM_SWITCH}" ] && die "No 'opam-switch' found in db '$DB' for build '$UUID'"
+[ -z "${OPAM_SWITCH}" ] && die "No 'opam-switch' found in db '${DB}' for build '${UUID}'"
 OPAM_SWITCH="${DATA_DIR}/${OPAM_SWITCH}"
 
 OPAM_GRAPH="opam-graph"
 MODULECTOMY="modulectomy"
 
-LATEST_TREEMAPVIZ_VERSION="$($MODULECTOMY --version)" || die "Failed to get modulectomy version"
+LATEST_TREEMAPVIZ_VERSION="$(${MODULECTOMY} --version)" || die "Failed to get modulectomy version"
-LATEST_DEPENDENCIESVIZ_VERSION="$($OPAM_GRAPH --version)" || die "Failed to get opam-graph version"
+LATEST_DEPENDENCIESVIZ_VERSION="$(${OPAM_GRAPH} --version)" || die "Failed to get opam-graph version"
 
 TREEMAP_CACHE_DIR="${CACHE_DIR}/treemap_${LATEST_TREEMAPVIZ_VERSION}"
 DEPENDENCIES_CACHE_DIR="${CACHE_DIR}/dependencies_${LATEST_DEPENDENCIESVIZ_VERSION}"
@@ -136,7 +136,7 @@ trap cleanup EXIT
 # /// Dependencies viz
 
 if [ ! -d "${DEPENDENCIES_CACHE_DIR}" ]; then
-    mkdir "${DEPENDENCIES_CACHE_DIR}" || die "Failed to create directory '$DEPENDENCIES_CACHE_DIR'"
+    mkdir "${DEPENDENCIES_CACHE_DIR}" || die "Failed to create directory '${DEPENDENCIES_CACHE_DIR}'"
 fi
 
 OPAM_SWITCH_FILEPATH='opam-switch'
@@ -144,8 +144,8 @@ OPAM_SWITCH_FILEPATH='opam-switch'
 get_opam_switch_hash () {
     sqlite3 "${DB}" "SELECT lower(hex(ba.sha256)) FROM build AS b
         JOIN build_artifact AS ba ON ba.build = b.id
-        WHERE uuid = '$UUID'
+        WHERE uuid = '${UUID}'
-        AND ba.filepath = '$OPAM_SWITCH_FILEPATH';"
+        AND ba.filepath = '${OPAM_SWITCH_FILEPATH}';"
 }
 
 DEPENDENCIES_INPUT_HASH="$(get_opam_switch_hash)" || die "Failed to get opam-switch hash from database"
@@ -155,7 +155,8 @@ if [ -e "${DEPENDENCIES_VIZ_FILENAME}" ]; then
     info "Dependency visualization already exists: '${DEPENDENCIES_VIZ_FILENAME}'"
 else
     if ${OPAM_GRAPH} --output-format=html "${OPAM_SWITCH}" > "${TMPDEPENDENCIES}"; then
-        mv "${TMPDEPENDENCIES}" "${DEPENDENCIES_VIZ_FILENAME}"
+        cp "${TMPDEPENDENCIES}" "${DEPENDENCIES_VIZ_FILENAME}"
+        rm "${TMPDEPENDENCIES}"
     else
         die "opam-graph failed to generate visualization"
     fi
@@ -173,16 +174,16 @@ stat_aux () {
     fi
 }
 
-SIZE="$(stat_aux "$BIN")"
+SIZE="$(stat_aux "${BIN}")"
 
 if [ ! -d "${TREEMAP_CACHE_DIR}" ]; then
-    mkdir "${TREEMAP_CACHE_DIR}" || die "Failed to create directory '$TREEMAP_CACHE_DIR'"
+    mkdir "${TREEMAP_CACHE_DIR}" || die "Failed to create directory '${TREEMAP_CACHE_DIR}'"
 fi
 
 get_debug_bin_hash () {
     sqlite3 "${DB}" "SELECT lower(hex(ba.sha256)) FROM build AS b
         JOIN build_artifact AS ba ON ba.build = b.id
-        WHERE uuid = '$UUID'
+        WHERE uuid = '${UUID}'
         AND ba.filepath LIKE '%.debug';"
 }
 
@@ -201,7 +202,8 @@ if [ -n "${DEBUG_BIN_RELATIVE}" ]; then
                 "${DEBUG_BIN}" \
                 > "${TMPTREE}"
         then
-            mv "${TMPTREE}" "${TREEMAP_VIZ_FILENAME}"
+            cp "${TMPTREE}" "${TREEMAP_VIZ_FILENAME}"
+            rm "${TMPTREE}"
         else
             die "modulectomy failed to generate visualization"
         fi