robur/builder-web
4
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

verify-data-dir-stream (#50)

Browse source 
builder-db verify-data-dir: stream build artifacts

Co-authored-by: Reynir Björnsson <reynir@reynir.dk>
Reviewed-on: https://git.robur.io/robur/builder-web/pulls/50
Co-Authored-By: hannes <hannes@mehnert.org>
Co-Committed-By: hannes <hannes@mehnert.org>
This commit is contained in:
Hannes Mehnert 2021-07-12 10:42:03 +00:00
parent 7c4bf56da6
commit e8f918230f
1 changed files with 28 additions and 18 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

46
bin/builder_db.ml
View file

@ -202,6 +202,12 @@ let verify_input_id () dbpath =
in in
or_die 1 r or_die 1 r
let num_build_artifacts =
Caqti_request.find
Caqti_type.unit
Caqti_type.int
"SELECT count(*) FROM build_artifact"
let build_artifacts : (unit, (Fpath.t * Fpath.t * Cstruct.t * int64) * [`build] Builder_db.Rep.id, [ `One | `Zero | `Many ]) Caqti_request.t = let build_artifacts : (unit, (Fpath.t * Fpath.t * Cstruct.t * int64) * [`build] Builder_db.Rep.id, [ `One | `Zero | `Many ]) Caqti_request.t =
Caqti_request.collect Caqti_request.collect
Caqti_type.unit Caqti_type.unit
@ -215,27 +221,31 @@ let verify_data_dir () datadir =
connect connect
(Uri.make ~scheme:"sqlite3" ~path:dbpath ~query:["create", ["false"]] ()) (Uri.make ~scheme:"sqlite3" ~path:dbpath ~query:["create", ["false"]] ())
>>= fun (module Db : Caqti_blocking.CONNECTION) -> >>= fun (module Db : Caqti_blocking.CONNECTION) ->
Db.collect_list build_artifacts () >>= fun build_artifacts -> Db.find num_build_artifacts () >>= fun num_build_artifacts ->
Logs.info (fun m -> m "total: %d artifacts" (List.length build_artifacts)); Logs.info (fun m -> m "total: %d artifacts" num_build_artifacts);
List.iteri (fun idx ((fpath, lpath, sha, size), _build_id) -> let progress =
if idx mod 100 = 0 then Logs.info (fun m -> m "%d" idx); let idx = ref 0 in
(match Fpath.segs lpath with fun () -> incr idx; if !idx mod 100 = 0 then Logs.info (fun m -> m "%d" !idx);
| _job :: _uuid :: output :: tl -> in
if output = "output" && Fpath.equal (Fpath.v (String.concat "/" tl)) fpath then Db.iter_s build_artifacts (fun ((fpath, lpath, sha, size), _build_id) ->
() progress ();
else (match Fpath.segs lpath with
Logs.err (fun m -> m "lpath (%a) and fpath (%a) do not match" Fpath.pp lpath Fpath.pp lpath) | _job :: _uuid :: "output" :: tl ->
| _ -> Logs.err (fun m -> m "lpath is not of form <job>/<uuid>/<output>/<filename>: %a" Fpath.pp lpath)); if Fpath.equal (Fpath.v (String.concat "/" tl)) fpath then
let abs_path = Fpath.(v datadir // lpath) in ()
(match Bos.OS.File.read abs_path with else
| Error (`Msg msg) -> Logs.err (fun m -> m "file %a not present: %s" Fpath.pp abs_path msg) Logs.err (fun m -> m "lpath (%a) and fpath (%a) do not match" Fpath.pp lpath Fpath.pp lpath)
| Ok data -> | _ -> Logs.err (fun m -> m "lpath is not of form <job>/<uuid>/output/<filename>: %a" Fpath.pp lpath));
let abs_path = Fpath.(v datadir // lpath) in
(match Bos.OS.File.read abs_path with
| Error (`Msg msg) -> Logs.err (fun m -> m "file %a not present: %s" Fpath.pp abs_path msg)
| Ok data ->
let s = Int64.of_int (String.length data) in let s = Int64.of_int (String.length data) in
if s <> size then Logs.err (fun m -> m "File %a has different size (in DB %Lu on disk %Lu)" Fpath.pp abs_path size s); if s <> size then Logs.err (fun m -> m "File %a has different size (in DB %Lu on disk %Lu)" Fpath.pp abs_path size s);
let sh = Mirage_crypto.Hash.SHA256.digest (Cstruct.of_string data) in let sh = Mirage_crypto.Hash.SHA256.digest (Cstruct.of_string data) in
if not (Cstruct.equal sha sh) then Logs.err (fun m -> m "File %a has different hash (in DB %a on disk %a" Fpath.pp abs_path Cstruct.hexdump_pp sha Cstruct.hexdump_pp sh)) if not (Cstruct.equal sha sh) then Logs.err (fun m -> m "File %a has different hash (in DB %a on disk %a" Fpath.pp abs_path Cstruct.hexdump_pp sha Cstruct.hexdump_pp sh)) ;
) build_artifacts; Ok ()
Ok () ) ()
in in
or_die 1 r or_die 1 r