remove README.md and build-hashes from build_artifact table and file system

no longer store full file on disk

bin/migrations/builder_migrations.ml

@@ -110,6 +110,11 @@ let f20210707d =
   Cmdliner.Term.(const do_database_action $ const M20210707d.fixup $ setup_log $ datadir),
   Cmdliner.Term.info ~doc "fixup-2021-07-07d"
 
+let f20210712b =
+  let doc = "Remove build-hashes and README from artifacts." in
+  Cmdliner.Term.(const do_database_action $ const M20210712b.fixup $ setup_log $ datadir),
+  Cmdliner.Term.info ~doc "fixup-2021-07-12b"
+
 let help_cmd =
   let topic =
     let doc = "Migration to get help on" in
@@ -148,6 +153,7 @@ let () =
         [ f20210707b ];
         [ f20210707c ];
         [ f20210707d ];
-        actions (module M20210712);
+        actions (module M20210712a);
+        [ f20210712b ];
       ])
   |> Cmdliner.Term.exit

bin/migrations/m20210712a.ml

@@ -1,5 +1,5 @@
 let new_version = 13L and old_version = 12L
-and identifier = "2021-07-12"
+and identifier = "2021-07-12a"
 and migrate_doc = "remove result_kind from build, add indexes idx_build_failed and idx_build_artifact_sha256"
 and rollback_doc = "add result_kind to build, remove indexes idx_build_failed and idx_build_artifact_sha256"
 

bin/migrations/m20210712b.ml

@@ -0,0 +1,23 @@
+
+let all_build_artifacts_like_hashes : (unit, [`build_artifact] Builder_db.Rep.id * Fpath.t, [ `Zero | `One | `Many ]) Caqti_request.t =
+  Caqti_request.collect ~oneshot:true
+    Caqti_type.unit
+    (Caqti_type.tup2 (Builder_db.Rep.id `build_artifact) Builder_db.Rep.fpath)
+    "SELECT id, localpath FROM build_artifact WHERE filepath LIKE '%.build-hashes'"
+
+let all_build_artifacts_like_readme : (unit, [`build_artifact] Builder_db.Rep.id * Fpath.t, [ `Zero | `One | `Many ]) Caqti_request.t =
+  Caqti_request.collect ~oneshot:true
+    Caqti_type.unit
+    (Caqti_type.tup2 (Builder_db.Rep.id `build_artifact) Builder_db.Rep.fpath)
+    "SELECT id, localpath FROM build_artifact WHERE filepath LIKE '%README.md'"
+
+let fixup datadir (module Db : Caqti_blocking.CONNECTION) =
+  let open Rresult.R.Infix in
+  Grej.check_version ~user_version:13L (module Db) >>= fun () ->
+  Db.rev_collect_list all_build_artifacts_like_hashes () >>= fun build_artifacts_build_hashes ->
+  Db.rev_collect_list all_build_artifacts_like_readme () >>= fun build_artifacts_readme ->
+  Grej.list_iter_result
+    (fun (artifact_id, artifact_lpath) ->
+        Bos.OS.File.delete (Fpath.append datadir artifact_lpath) >>= fun () ->
+        Db.exec Builder_db.Build_artifact.remove artifact_id)
+    (build_artifacts_build_hashes @ build_artifacts_readme)

lib/model.ml

@@ -182,10 +182,6 @@ let save file data =
           Lwt_result.fail (`Msg (Unix.error_message e))
       | e -> Lwt.fail e)
 
-let save_exec build_dir exec =
-  let cs = Builder.Asn.exec_to_cs exec in
-  save Fpath.(build_dir / "full") (Cstruct.to_string cs)
-
 let save_file dir staging (filepath, data) =
   let size = String.length data in
   let sha256 = Mirage_crypto.Hash.SHA256.digest (Cstruct.of_string data) in
@@ -204,7 +200,7 @@ let save_files dir staging files =
     (Lwt_result.return [])
     files
 
-let save_all staging_dir (((job : Builder.script_job), uuid, _, _, _, _, artifacts) as exec) =
+let save_all staging_dir (job : Builder.script_job) uuid artifacts =
   let build_dir = Fpath.(v job.Builder.name / Uuidm.to_string uuid) in
   let output_dir = Fpath.(build_dir / "output")
   and staging_output_dir = Fpath.(staging_dir / "output") in
@@ -213,7 +209,6 @@ let save_all staging_dir (((job : Builder.script_job), uuid, _, _, _, _, artifac
       then Lwt_result.fail (`Msg "build directory already exists")
       else Lwt_result.return ()) >>= fun () ->
   Lwt.return (Bos.OS.Dir.create staging_output_dir) >>= fun _ ->
-  save_exec staging_dir exec >>= fun () ->
   save_files output_dir staging_output_dir artifacts >>= fun artifacts ->
   Lwt_result.return artifacts
 
@@ -284,7 +279,7 @@ let compute_input_id artifacts =
 let add_build
     datadir
     user_id
-    (((job : Builder.script_job), uuid, console, start, finish, result, raw_artifacts) as exec)
+    ((job : Builder.script_job), uuid, console, start, finish, result, raw_artifacts)
     (module Db : CONN) =
   let open Builder_db in
   let job_name = job.Builder.name in
@@ -299,7 +294,13 @@ let add_build
         e)
       x
   in
-  or_cleanup (save_all staging_dir exec) >>= fun artifacts ->
+  let artifacts_to_preserve =
+    let not_interesting p =
+      String.equal (Fpath.basename p) "README.md" || String.equal (Fpath.get_ext p) ".build-hashes"
+    in
+    List.filter (fun (p, _) -> not (not_interesting p)) raw_artifacts
+  in
+  or_cleanup (save_all staging_dir job uuid artifacts_to_preserve) >>= fun artifacts ->
   let r =
     Db.start () >>= fun () ->
     Db.exec Job.try_add job_name >>= fun () ->