From b279eb521bdb5a9e59f669f4ed8b1b467e43b432 Mon Sep 17 00:00:00 2001 From: Robur Date: Fri, 25 Jun 2021 10:26:03 +0000 Subject: [PATCH] Adapt to build input files removal * Do not synthesize empty input files list * Drop table build_file --- bin/migrations/builder_migrations.ml | 1 + bin/migrations/m20210625.ml | 37 ++++++++++++++++++ db/builder_db.ml | 57 +--------------------------- db/builder_db.mli | 18 --------- lib/builder_web.ml | 2 +- lib/model.ml | 14 +------ 6 files changed, 42 insertions(+), 87 deletions(-) create mode 100644 bin/migrations/m20210625.ml diff --git a/bin/migrations/builder_migrations.ml b/bin/migrations/builder_migrations.ml index 04c3fd8..06d35b4 100644 --- a/bin/migrations/builder_migrations.ml +++ b/bin/migrations/builder_migrations.ml @@ -119,5 +119,6 @@ let () = actions (module M20210602); actions (module M20210608); actions (module M20210609); + actions (module M20210625); ]) |> Cmdliner.Term.exit diff --git a/bin/migrations/m20210625.ml b/bin/migrations/m20210625.ml new file mode 100644 index 0000000..906c731 --- /dev/null +++ b/bin/migrations/m20210625.ml @@ -0,0 +1,37 @@ +let new_version = 8L and old_version = 7L +let identifier = "2021-06-25" +let migrate_doc = "drop build_file table" +let rollback_doc = "recreate build_file table" + +let build_file = + Caqti_request.exec + Caqti_type.unit + {| CREATE TABLE build_file ( + id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, + filepath TEXT NOT NULL, -- the path as in the build + localpath TEXT NOT NULL, -- local path to the file on disk + sha256 BLOB NOT NULL, + size INTEGER NOT NULL, + build INTEGER NOT NULL, + + FOREIGN KEY(build) REFERENCES build(id), + UNIQUE(build, filepath) + ) + |} + +let drop_build_file = + Caqti_request.exec + Caqti_type.unit + "DROP TABLE build_file" + +open Rresult.R.Infix + +let migrate _datadir (module Db : Caqti_blocking.CONNECTION) = + Grej.check_version ~user_version:old_version (module Db) >>= fun () -> + Db.exec drop_build_file () >>= fun () -> + Db.exec (Grej.set_version new_version) () + +let rollback _datadir (module Db : Caqti_blocking.CONNECTION) = + Grej.check_version ~user_version:new_version (module Db) >>= fun () -> + Db.exec build_file () >>= fun () -> + Db.exec (Grej.set_version old_version) () diff --git a/db/builder_db.ml b/db/builder_db.ml index 6538c59..bfb2dee 100644 --- a/db/builder_db.ml +++ b/db/builder_db.ml @@ -4,7 +4,7 @@ open Rep let application_id = 1234839235l (* Please update this when making changes! *) -let current_version = 7L +let current_version = 8L type id = Rep.id @@ -150,59 +150,6 @@ module Build_artifact = struct "DELETE FROM build_artifact WHERE build = ?" end -module Build_file = struct - let migrate = - Caqti_request.exec - Caqti_type.unit - {| CREATE TABLE build_file ( - id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, - filepath TEXT NOT NULL, -- the path as in the build - localpath TEXT NOT NULL, -- local path to the file on disk - sha256 BLOB NOT NULL, - size INTEGER NOT NULL, - build INTEGER NOT NULL, - - FOREIGN KEY(build) REFERENCES build(id), - UNIQUE(build, filepath) - ) - |} - - let rollback = - Caqti_request.exec - Caqti_type.unit - "DROP TABLE IF EXISTS build_file" - - let get_by_build_uuid = - Caqti_request.find_opt - (Caqti_type.tup2 uuid fpath) - (Caqti_type.tup2 id file) - {| SELECT build_file.id, build_file.localpath, - build_file.localpath, build_file.sha256, build_file.size - FROM build_file - INNER JOIN build ON build.id = build_file.build - WHERE build.uuid = ? AND build_file.filepath = ? - |} - - let get_all_by_build = - Caqti_request.collect - id - Caqti_type.(tup2 - id - file) - "SELECT id, filepath, localpath, sha256, size FROM build_file WHERE build = ?" - - let add = - Caqti_request.exec - Caqti_type.(tup2 file id) - "INSERT INTO build_file (filepath, localpath, sha256, size, build) - VALUES (?, ?, ?, ?, ?)" - - let remove_by_build = - Caqti_request.exec - id - "DELETE FROM build_file WHERE build = ?" -end - module Build = struct type t = { uuid : Uuidm.t; @@ -568,7 +515,6 @@ let migrate = [ Job.migrate; Build.migrate; Build_artifact.migrate; - Build_file.migrate; User.migrate; Access_list.migrate; Caqti_request.exec Caqti_type.unit @@ -580,7 +526,6 @@ let migrate = [ let rollback = [ Access_list.rollback; User.rollback; - Build_file.migrate; Build_artifact.rollback; Build.rollback; Job.rollback; diff --git a/db/builder_db.mli b/db/builder_db.mli index c8bbdd2..c2bc021 100644 --- a/db/builder_db.mli +++ b/db/builder_db.mli @@ -85,24 +85,6 @@ module Build_artifact : sig (id, unit, [< `Many | `One | `Zero > `Zero ]) Caqti_request.t end -module Build_file : sig - val migrate : - (unit, unit, [< `Many | `One | `Zero > `Zero ]) Caqti_request.t - val rollback : - (unit, unit, [< `Many | `One | `Zero > `Zero ]) Caqti_request.t - - val get_by_build_uuid : - (Uuidm.t * Fpath.t, id * file, - [< `Many | `One | `Zero > `One `Zero ]) - Caqti_request.t - val get_all_by_build : - (id, id * file, [ `Many | `One | `Zero ]) Caqti_request.t - val add : - (file * id, unit, [< `Many | `One | `Zero > `Zero ]) Caqti_request.t - val remove_by_build : - (id, unit, [< `Many | `One | `Zero > `Zero ]) Caqti_request.t -end - module Build : sig type t = { diff --git a/lib/builder_web.ml b/lib/builder_web.ml index 64e31b5..c5a6711 100644 --- a/lib/builder_web.ml +++ b/lib/builder_web.ml @@ -262,7 +262,7 @@ let add_routes datadir = let datadir = Dream.global datadir_global req in let exec = let now = Ptime_clock.now () in - ({ Builder.name = job ; script = "" ; files = [] }, uuid, [], now, now, Builder.Exited 0, + ({ Builder.name = job ; script = "" }, uuid, [], now, now, Builder.Exited 0, [ (Fpath.(v "bin" / job + "bin"), body) ]) in (Lwt.return (Dream.local Authorization.user_info_local req |> diff --git a/lib/model.ml b/lib/model.ml index ddc538b..a2926c0 100644 --- a/lib/model.ml +++ b/lib/model.ml @@ -168,20 +168,16 @@ let save_files dir staging files = let save_all staging_dir ((job, uuid, _, _, _, _, artifacts) as exec) = let build_dir = Fpath.(v job.Builder.name / Uuidm.to_string uuid) in - let input_dir = Fpath.(build_dir / "input") - and staging_input_dir = Fpath.(staging_dir / "input") in let output_dir = Fpath.(build_dir / "output") and staging_output_dir = Fpath.(staging_dir / "output") in Lwt.return (Bos.OS.Dir.create staging_dir) >>= (fun created -> if not created then Lwt_result.fail (`Msg "build directory already exists") else Lwt_result.return ()) >>= fun () -> - Lwt.return (Bos.OS.Dir.create staging_input_dir) >>= fun _ -> Lwt.return (Bos.OS.Dir.create staging_output_dir) >>= fun _ -> save_exec staging_dir exec >>= fun () -> save_files output_dir staging_output_dir artifacts >>= fun artifacts -> - save_files input_dir staging_input_dir job.Builder.files >>= fun input_files -> - Lwt_result.return (artifacts, input_files) + Lwt_result.return artifacts let commit_files datadir staging_dir job_name uuid = let job_dir = Fpath.(datadir / job_name) in @@ -207,7 +203,7 @@ let add_build e) x in - or_cleanup (save_all staging_dir exec) >>= fun (artifacts, input_files) -> + or_cleanup (save_all staging_dir exec) >>= fun artifacts -> let r = Db.start () >>= fun () -> Db.exec Job.try_add job_name >>= fun () -> @@ -223,12 +219,6 @@ let add_build Db.exec Build_artifact.add (file, id)) (Lwt_result.return ()) artifacts >>= fun () -> - List.fold_left - (fun r file -> - r >>= fun () -> - Db.exec Build_file.add (file, id)) - (Lwt_result.return ()) - input_files >>= fun () -> Db.collect_list Build_artifact.get_all_by_build id >>= fun artifacts -> (match List.filter (fun (_, p) -> Fpath.(is_prefix (v "bin/") p.filepath)) artifacts with | [ (build_artifact_id, _) ] -> Db.exec Build.set_main_binary (id, build_artifact_id)