robur/builder-web
4
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

remove README.md and build-hashes from build_artifact table and file system

Browse source 
no longer store full file on disk
This commit is contained in:
Robur 2021-07-12 14:10:54 +00:00
parent 928821fec6
commit 392286dd98
4 changed files with 40 additions and 10 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

8
bin/migrations/builder_migrations.ml
View file

@ -110,6 +110,11 @@ let f20210707d =
Cmdliner.Term.(const do_database_action $ const M20210707d.fixup $ setup_log $ datadir),
Cmdliner.Term.info ~doc "fixup-2021-07-07d"
let f20210712b =
let doc = "Remove build-hashes and README from artifacts." in
Cmdliner.Term.(const do_database_action $ const M20210712b.fixup $ setup_log $ datadir),
Cmdliner.Term.info ~doc "fixup-2021-07-12b"
let help_cmd =
let topic =
let doc = "Migration to get help on" in
@ -148,6 +153,7 @@ let () =
[ f20210707b ];
[ f20210707c ];
[ f20210707d ];
actions (module M20210712);
actions (module M20210712a);
[ f20210712b ];
])
|> Cmdliner.Term.exit

2
bin/migrations/m20210712.ml → bin/migrations/m20210712a.ml
View file

@ -1,5 +1,5 @@
let new_version = 13L and old_version = 12L
and identifier = "2021-07-12"
and identifier = "2021-07-12a"
and migrate_doc = "remove result_kind from build, add indexes idx_build_failed and idx_build_artifact_sha256"
and rollback_doc = "add result_kind to build, remove indexes idx_build_failed and idx_build_artifact_sha256"

23
bin/migrations/m20210712b.ml Normal file
View file

@ -0,0 +1,23 @@
let all_build_artifacts_like_hashes : (unit, [`build_artifact] Builder_db.Rep.id * Fpath.t, [ `Zero | `One | `Many ]) Caqti_request.t =
Caqti_request.collect ~oneshot:true
Caqti_type.unit
(Caqti_type.tup2 (Builder_db.Rep.id `build_artifact) Builder_db.Rep.fpath)
"SELECT id, localpath FROM build_artifact WHERE filepath LIKE '%.build-hashes'"
let all_build_artifacts_like_readme : (unit, [`build_artifact] Builder_db.Rep.id * Fpath.t, [ `Zero | `One | `Many ]) Caqti_request.t =
Caqti_request.collect ~oneshot:true
Caqti_type.unit
(Caqti_type.tup2 (Builder_db.Rep.id `build_artifact) Builder_db.Rep.fpath)
"SELECT id, localpath FROM build_artifact WHERE filepath LIKE '%README.md'"
let fixup datadir (module Db : Caqti_blocking.CONNECTION) =
let open Rresult.R.Infix in
Grej.check_version ~user_version:13L (module Db) >>= fun () ->
Db.rev_collect_list all_build_artifacts_like_hashes () >>= fun build_artifacts_build_hashes ->
Db.rev_collect_list all_build_artifacts_like_readme () >>= fun build_artifacts_readme ->
Grej.list_iter_result
(fun (artifact_id, artifact_lpath) ->
Bos.OS.File.delete (Fpath.append datadir artifact_lpath) >>= fun () ->
Db.exec Builder_db.Build_artifact.remove artifact_id)
(build_artifacts_build_hashes @ build_artifacts_readme)

17
lib/model.ml
View file

@ -182,10 +182,6 @@ let save file data =
Lwt_result.fail (`Msg (Unix.error_message e))
| e -> Lwt.fail e)
let save_exec build_dir exec =
let cs = Builder.Asn.exec_to_cs exec in
save Fpath.(build_dir / "full") (Cstruct.to_string cs)
let save_file dir staging (filepath, data) =
let size = String.length data in
let sha256 = Mirage_crypto.Hash.SHA256.digest (Cstruct.of_string data) in
@ -204,7 +200,7 @@ let save_files dir staging files =
(Lwt_result.return [])
files
let save_all staging_dir (((job : Builder.script_job), uuid, _, _, _, _, artifacts) as exec) =
let save_all staging_dir (job : Builder.script_job) uuid artifacts =
let build_dir = Fpath.(v job.Builder.name / Uuidm.to_string uuid) in
let output_dir = Fpath.(build_dir / "output")
and staging_output_dir = Fpath.(staging_dir / "output") in
@ -213,7 +209,6 @@ let save_all staging_dir (((job : Builder.script_job), uuid, _, _, _, _, artifac
then Lwt_result.fail (`Msg "build directory already exists")
else Lwt_result.return ()) >>= fun () ->
Lwt.return (Bos.OS.Dir.create staging_output_dir) >>= fun _ ->
save_exec staging_dir exec >>= fun () ->
save_files output_dir staging_output_dir artifacts >>= fun artifacts ->
Lwt_result.return artifacts
@ -284,7 +279,7 @@ let compute_input_id artifacts =
let add_build
datadir
user_id
(((job : Builder.script_job), uuid, console, start, finish, result, raw_artifacts) as exec)
((job : Builder.script_job), uuid, console, start, finish, result, raw_artifacts)
(module Db : CONN) =
let open Builder_db in
let job_name = job.Builder.name in
@ -299,7 +294,13 @@ let add_build
e)
x
in
or_cleanup (save_all staging_dir exec) >>= fun artifacts ->
let artifacts_to_preserve =
let not_interesting p =
String.equal (Fpath.basename p) "README.md" || String.equal (Fpath.get_ext p) ".build-hashes"
in
List.filter (fun (p, _) -> not (not_interesting p)) raw_artifacts
in
or_cleanup (save_all staging_dir job uuid artifacts_to_preserve) >>= fun artifacts ->
let r =
Db.start () >>= fun () ->
Db.exec Job.try_add job_name >>= fun () ->