From 392286dd98fbc17cdabc58341afa16769d9968c5 Mon Sep 17 00:00:00 2001 From: Robur Date: Mon, 12 Jul 2021 14:10:54 +0000 Subject: [PATCH] remove README.md and build-hashes from build_artifact table and file system no longer store full file on disk --- bin/migrations/builder_migrations.ml | 8 ++++++- .../{m20210712.ml => m20210712a.ml} | 2 +- bin/migrations/m20210712b.ml | 23 +++++++++++++++++++ lib/model.ml | 17 +++++++------- 4 files changed, 40 insertions(+), 10 deletions(-) rename bin/migrations/{m20210712.ml => m20210712a.ml} (99%) create mode 100644 bin/migrations/m20210712b.ml diff --git a/bin/migrations/builder_migrations.ml b/bin/migrations/builder_migrations.ml index befee26..8f39ac4 100644 --- a/bin/migrations/builder_migrations.ml +++ b/bin/migrations/builder_migrations.ml @@ -110,6 +110,11 @@ let f20210707d = Cmdliner.Term.(const do_database_action $ const M20210707d.fixup $ setup_log $ datadir), Cmdliner.Term.info ~doc "fixup-2021-07-07d" +let f20210712b = + let doc = "Remove build-hashes and README from artifacts." in + Cmdliner.Term.(const do_database_action $ const M20210712b.fixup $ setup_log $ datadir), + Cmdliner.Term.info ~doc "fixup-2021-07-12b" + let help_cmd = let topic = let doc = "Migration to get help on" in @@ -148,6 +153,7 @@ let () = [ f20210707b ]; [ f20210707c ]; [ f20210707d ]; - actions (module M20210712); + actions (module M20210712a); + [ f20210712b ]; ]) |> Cmdliner.Term.exit diff --git a/bin/migrations/m20210712.ml b/bin/migrations/m20210712a.ml similarity index 99% rename from bin/migrations/m20210712.ml rename to bin/migrations/m20210712a.ml index aabf984..3efec10 100644 --- a/bin/migrations/m20210712.ml +++ b/bin/migrations/m20210712a.ml @@ -1,5 +1,5 @@ let new_version = 13L and old_version = 12L -and identifier = "2021-07-12" +and identifier = "2021-07-12a" and migrate_doc = "remove result_kind from build, add indexes idx_build_failed and idx_build_artifact_sha256" and rollback_doc = "add result_kind to build, remove indexes idx_build_failed and idx_build_artifact_sha256" diff --git a/bin/migrations/m20210712b.ml b/bin/migrations/m20210712b.ml new file mode 100644 index 0000000..0c26dbb --- /dev/null +++ b/bin/migrations/m20210712b.ml @@ -0,0 +1,23 @@ + +let all_build_artifacts_like_hashes : (unit, [`build_artifact] Builder_db.Rep.id * Fpath.t, [ `Zero | `One | `Many ]) Caqti_request.t = + Caqti_request.collect ~oneshot:true + Caqti_type.unit + (Caqti_type.tup2 (Builder_db.Rep.id `build_artifact) Builder_db.Rep.fpath) + "SELECT id, localpath FROM build_artifact WHERE filepath LIKE '%.build-hashes'" + +let all_build_artifacts_like_readme : (unit, [`build_artifact] Builder_db.Rep.id * Fpath.t, [ `Zero | `One | `Many ]) Caqti_request.t = + Caqti_request.collect ~oneshot:true + Caqti_type.unit + (Caqti_type.tup2 (Builder_db.Rep.id `build_artifact) Builder_db.Rep.fpath) + "SELECT id, localpath FROM build_artifact WHERE filepath LIKE '%README.md'" + +let fixup datadir (module Db : Caqti_blocking.CONNECTION) = + let open Rresult.R.Infix in + Grej.check_version ~user_version:13L (module Db) >>= fun () -> + Db.rev_collect_list all_build_artifacts_like_hashes () >>= fun build_artifacts_build_hashes -> + Db.rev_collect_list all_build_artifacts_like_readme () >>= fun build_artifacts_readme -> + Grej.list_iter_result + (fun (artifact_id, artifact_lpath) -> + Bos.OS.File.delete (Fpath.append datadir artifact_lpath) >>= fun () -> + Db.exec Builder_db.Build_artifact.remove artifact_id) + (build_artifacts_build_hashes @ build_artifacts_readme) diff --git a/lib/model.ml b/lib/model.ml index 506061f..05f18f5 100644 --- a/lib/model.ml +++ b/lib/model.ml @@ -182,10 +182,6 @@ let save file data = Lwt_result.fail (`Msg (Unix.error_message e)) | e -> Lwt.fail e) -let save_exec build_dir exec = - let cs = Builder.Asn.exec_to_cs exec in - save Fpath.(build_dir / "full") (Cstruct.to_string cs) - let save_file dir staging (filepath, data) = let size = String.length data in let sha256 = Mirage_crypto.Hash.SHA256.digest (Cstruct.of_string data) in @@ -204,7 +200,7 @@ let save_files dir staging files = (Lwt_result.return []) files -let save_all staging_dir (((job : Builder.script_job), uuid, _, _, _, _, artifacts) as exec) = +let save_all staging_dir (job : Builder.script_job) uuid artifacts = let build_dir = Fpath.(v job.Builder.name / Uuidm.to_string uuid) in let output_dir = Fpath.(build_dir / "output") and staging_output_dir = Fpath.(staging_dir / "output") in @@ -213,7 +209,6 @@ let save_all staging_dir (((job : Builder.script_job), uuid, _, _, _, _, artifac then Lwt_result.fail (`Msg "build directory already exists") else Lwt_result.return ()) >>= fun () -> Lwt.return (Bos.OS.Dir.create staging_output_dir) >>= fun _ -> - save_exec staging_dir exec >>= fun () -> save_files output_dir staging_output_dir artifacts >>= fun artifacts -> Lwt_result.return artifacts @@ -284,7 +279,7 @@ let compute_input_id artifacts = let add_build datadir user_id - (((job : Builder.script_job), uuid, console, start, finish, result, raw_artifacts) as exec) + ((job : Builder.script_job), uuid, console, start, finish, result, raw_artifacts) (module Db : CONN) = let open Builder_db in let job_name = job.Builder.name in @@ -299,7 +294,13 @@ let add_build e) x in - or_cleanup (save_all staging_dir exec) >>= fun artifacts -> + let artifacts_to_preserve = + let not_interesting p = + String.equal (Fpath.basename p) "README.md" || String.equal (Fpath.get_ext p) ".build-hashes" + in + List.filter (fun (p, _) -> not (not_interesting p)) raw_artifacts + in + or_cleanup (save_all staging_dir job uuid artifacts_to_preserve) >>= fun artifacts -> let r = Db.start () >>= fun () -> Db.exec Job.try_add job_name >>= fun () ->