From e8f918230f513c66c858d784d38faee8e9d9e9e3 Mon Sep 17 00:00:00 2001
From: hannes <hannes@mehnert.org>
Date: Mon, 12 Jul 2021 10:42:03 +0000
Subject: [PATCH] verify-data-dir-stream (#50)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

builder-db verify-data-dir: stream build artifacts

Co-authored-by: Reynir Björnsson <reynir@reynir.dk>
Reviewed-on: https://git.robur.io/robur/builder-web/pulls/50
Co-Authored-By: hannes <hannes@mehnert.org>
Co-Committed-By: hannes <hannes@mehnert.org>
---
 bin/builder_db.ml | 46 ++++++++++++++++++++++++++++------------------
 1 file changed, 28 insertions(+), 18 deletions(-)

diff --git a/bin/builder_db.ml b/bin/builder_db.ml
index 10b8e91..e6024cf 100644
--- a/bin/builder_db.ml
+++ b/bin/builder_db.ml
@@ -202,6 +202,12 @@ let verify_input_id () dbpath =
    in
    or_die 1 r
 
+let num_build_artifacts =
+  Caqti_request.find
+    Caqti_type.unit
+    Caqti_type.int
+    "SELECT count(*) FROM build_artifact"
+
 let build_artifacts : (unit, (Fpath.t * Fpath.t * Cstruct.t * int64) * [`build] Builder_db.Rep.id, [ `One | `Zero | `Many ]) Caqti_request.t =
   Caqti_request.collect
     Caqti_type.unit
@@ -215,27 +221,31 @@ let verify_data_dir () datadir =
     connect
       (Uri.make ~scheme:"sqlite3" ~path:dbpath ~query:["create", ["false"]] ())
     >>= fun (module Db : Caqti_blocking.CONNECTION)  ->
-    Db.collect_list build_artifacts () >>= fun build_artifacts ->
-    Logs.info (fun m -> m "total: %d artifacts" (List.length build_artifacts));
-    List.iteri (fun idx ((fpath, lpath, sha, size), _build_id) ->
-      if idx mod 100 = 0 then Logs.info (fun m -> m "%d" idx);
-      (match Fpath.segs lpath with
-       | _job :: _uuid :: output :: tl ->
-         if output = "output" && Fpath.equal (Fpath.v (String.concat "/" tl)) fpath then
-           ()
-         else
-           Logs.err (fun m -> m "lpath (%a) and fpath (%a) do not match" Fpath.pp lpath Fpath.pp lpath)
-       | _ -> Logs.err (fun m -> m "lpath is not of form <job>/<uuid>/<output>/<filename>: %a" Fpath.pp lpath));
-      let abs_path = Fpath.(v datadir // lpath) in
-      (match Bos.OS.File.read abs_path with
-       | Error (`Msg msg) -> Logs.err (fun m -> m "file %a not present: %s" Fpath.pp abs_path msg)
-       | Ok data ->
+    Db.find num_build_artifacts () >>= fun num_build_artifacts ->
+    Logs.info (fun m -> m "total: %d artifacts" num_build_artifacts);
+    let progress =
+      let idx = ref 0 in
+      fun () -> incr idx; if !idx mod 100 = 0 then Logs.info (fun m -> m "%d" !idx);
+    in
+    Db.iter_s build_artifacts (fun ((fpath, lpath, sha, size), _build_id) ->
+        progress ();
+        (match Fpath.segs lpath with
+         | _job :: _uuid :: "output" :: tl ->
+           if Fpath.equal (Fpath.v (String.concat "/" tl)) fpath then
+             ()
+           else
+             Logs.err (fun m -> m "lpath (%a) and fpath (%a) do not match" Fpath.pp lpath Fpath.pp lpath)
+         | _ -> Logs.err (fun m -> m "lpath is not of form <job>/<uuid>/output/<filename>: %a" Fpath.pp lpath));
+        let abs_path = Fpath.(v datadir // lpath) in
+        (match Bos.OS.File.read abs_path with
+         | Error (`Msg msg) -> Logs.err (fun m -> m "file %a not present: %s" Fpath.pp abs_path msg)
+         | Ok data ->
            let s = Int64.of_int (String.length data) in
            if s <> size then Logs.err (fun m -> m "File %a has different size (in DB %Lu on disk %Lu)" Fpath.pp abs_path size s);
            let sh = Mirage_crypto.Hash.SHA256.digest (Cstruct.of_string data) in
-           if not (Cstruct.equal sha sh) then Logs.err (fun m -> m "File %a has different hash (in DB %a on disk %a" Fpath.pp abs_path Cstruct.hexdump_pp sha Cstruct.hexdump_pp sh))
-    ) build_artifacts;
-    Ok ()
+           if not (Cstruct.equal sha sh) then Logs.err (fun m -> m "File %a has different hash (in DB %a on disk %a" Fpath.pp abs_path Cstruct.hexdump_pp sha Cstruct.hexdump_pp sh)) ;
+        Ok ()
+      ) ()
   in
   or_die 1 r