open Lwt_result.Infix let src = Logs.Src.create "builder-web.model" ~doc:"Builder_web model" module Log = (val Logs.src_log src : Logs.LOG) module type CONN = Caqti_lwt.CONNECTION type error = [ Caqti_error.call_or_retrieve | `Not_found | `File_error of Fpath.t | `Msg of string ] let pp_error ppf = function | `Not_found -> Format.fprintf ppf "value not found in database" | `File_error path -> Format.fprintf ppf "error reading file %a" Fpath.pp path | `Msg e -> Format.fprintf ppf "error %s" e | #Caqti_error.call_or_retrieve as e -> Caqti_error.pp ppf e let not_found = function | None -> Lwt_result.fail `Not_found | Some v -> Lwt_result.return v let staging datadir = Fpath.(datadir / "_staging") let artifact_path artifact = let (`Hex sha256) = Hex.of_cstruct artifact.Builder_db.sha256 in (* NOTE: [sha256] is 64 characters when it's a hex sha256 checksum *) (* NOTE: We add the prefix to reduce the number of files in a directory - a workaround for inferior filesystems. We can easily revert this by changing this function and adding a migration. *) let prefix = String.sub sha256 0 2 in Fpath.(v "_artifacts" / prefix / sha256) let read_file datadir filepath = let filepath = Fpath.(datadir // filepath) in Lwt.try_bind (fun () -> Lwt_io.open_file ~mode:Lwt_io.Input (Fpath.to_string filepath)) (fun ic -> let open Lwt.Infix in Lwt_io.read ic >>= fun data -> Lwt_io.close ic >>= fun () -> Lwt_result.return data) (function | Unix.Unix_error (e, _, _) -> Log.warn (fun m -> m "Error reading local file %a: %s" Fpath.pp filepath (Unix.error_message e)); Lwt.return_error (`File_error filepath) | e -> Lwt.fail e) let build_artifact build filepath (module Db : CONN) = Db.find_opt Builder_db.Build_artifact.get_by_build_uuid (build, filepath) >>= not_found >|= snd let build_artifact_by_id id (module Db : CONN) = Db.find Builder_db.Build_artifact.get id let build_artifact_data datadir file = read_file datadir (artifact_path file) let build_artifacts build (module Db : CONN) = Db.collect_list Builder_db.Build_artifact.get_all_by_build build >|= List.map snd let solo5_manifest datadir file = let buf = Owee_buf.map_binary Fpath.(to_string (datadir // artifact_path file)) in Solo5_elftool.query_manifest buf |> Result.to_option let platforms_of_job id (module Db : CONN) = Db.collect_list Builder_db.Build.get_platforms_for_job id let build uuid (module Db : CONN) = Db.find_opt Builder_db.Build.get_by_uuid uuid >>= not_found let build_with_main_binary job platform (module Db : CONN) = Db.find_opt Builder_db.Build.get_latest_successful_with_binary (job, platform) >|= Option.map (fun (_id, build, file) -> (build, file)) let build_hash hash (module Db : CONN) = Db.find_opt Builder_db.Build.get_with_jobname_by_hash hash let build_exists uuid (module Db : CONN) = Db.find_opt Builder_db.Build.get_by_uuid uuid >|= Option.is_some let latest_successful_build job_id platform (module Db : CONN) = Db.find_opt Builder_db.Build.get_latest_successful (job_id, platform) let latest_successful_build_uuid job_id platform db = latest_successful_build job_id platform db >|= fun build -> Option.map (fun build -> build.Builder_db.Build.uuid) build let previous_successful_build_different_output id (module Db : CONN) = Db.find_opt Builder_db.Build.get_previous_successful_different_output id let next_successful_build_different_output id (module Db : CONN) = Db.find_opt Builder_db.Build.get_next_successful_different_output id let failed_builds ~start ~count platform (module Db : CONN) = Db.collect_list Builder_db.Build.get_all_failed (start, count, platform) let builds_with_different_input_and_same_main_binary id (module Db : CONN) = Db.collect_list Builder_db.Build.get_different_input_same_output_input_ids id >>= fun ids -> Lwt_list.fold_left_s (fun acc input_id -> match acc with | Error _ as e -> Lwt.return e | Ok metas -> Db.find Builder_db.Build.get_one_by_input_id input_id >>= fun build -> Lwt.return (Ok (build :: metas))) (Ok []) ids let builds_with_same_input_and_same_main_binary id (module Db : CONN) = Db.collect_list Builder_db.Build.get_same_input_same_output_builds id let builds_with_same_input_and_different_main_binary id (module Db : CONN) = Db.collect_list Builder_db.Build.get_same_input_different_output_hashes id >>= fun hashes -> Lwt_list.fold_left_s (fun acc hash -> match acc with | Error _ as e -> Lwt.return e | Ok metas -> Db.find Builder_db.Build.get_by_hash hash >>= fun build -> Lwt.return (Ok (build :: metas))) (Ok []) hashes let build_console_by_uuid datadir uuid (module Db : CONN) = build uuid (module Db) >>= fun (_id, { Builder_db.Build.console; _ })-> read_file datadir console let build_script_by_uuid datadir uuid (module Db : CONN) = build uuid (module Db) >>= fun (_id, { Builder_db.Build.script; _ })-> read_file datadir script let job_id job_name (module Db : CONN) = Db.find_opt Builder_db.Job.get_id_by_name job_name let readme job (module Db : CONN) = job_id job (module Db) >>= not_found >>= fun job_id -> Db.find Builder_db.Tag.get_id_by_name "readme.md" >>= fun readme_id -> Db.find_opt Builder_db.Job_tag.get_value (readme_id, job_id) let job_and_readme job (module Db : CONN) = job_id job (module Db) >>= not_found >>= fun job_id -> Db.find Builder_db.Tag.get_id_by_name "readme.md" >>= fun readme_id -> Db.find_opt Builder_db.Job_tag.get_value (readme_id, job_id) >|= fun readme -> job_id, readme let builds_grouped_by_output job_id platform (module Db : CONN) = Db.collect_list Builder_db.Build.get_all_artifact_sha (job_id, platform) >>= fun sha -> Lwt_list.fold_left_s (fun acc hash -> match acc with | Error _ as e -> Lwt.return e | Ok builds -> Db.find Builder_db.Build.get_with_main_binary_by_hash hash >|= fun b -> b :: builds) (Ok []) sha >|= List.rev let builds_grouped_by_output_with_failed job_id platform ((module Db : CONN) as db) = builds_grouped_by_output job_id platform db >>= fun builds -> Db.collect_list Builder_db.Build.get_failed_builds (job_id, platform) >|= fun failed -> let failed = List.map (fun b -> b, None) failed in let cmp (a, _) (b, _) = Ptime.compare b.Builder_db.Build.start a.Builder_db.Build.start in List.merge cmp builds failed let jobs_with_section_synopsis (module Db : CONN) = Db.collect_list Builder_db.Job.get_all_with_section_synopsis () let job_name id (module Db : CONN) = Db.find Builder_db.Job.get id let user username (module Db : CONN) = Db.find_opt Builder_db.User.get_user username let authorized user_id job_name (module Db : CONN) = job_id job_name (module Db) >>= (function None -> Lwt_result.fail (`Msg "No such job") | Some r -> Lwt_result.return r) >>= fun job_id -> Db.find Builder_db.Access_list.get (user_id, job_id) >|= fun _id -> () let cleanup_staging datadir (module Db : Caqti_lwt.CONNECTION) = let cleanup_staged staged = match Uuidm.of_string (Fpath.to_string staged) with | None -> Log.warn (fun m -> m "Non-uuid staged files: %a" Fpath.pp Fpath.(staging datadir // staged)); Lwt.return (Bos.OS.Path.delete ~recurse:true Fpath.(staging datadir // staged)) | Some uuid -> let staged = Fpath.(staging datadir // staged) in Db.find_opt Builder_db.Build.get_by_uuid uuid >>= function | Some (_id, build) -> Db.find Builder_db.Job.get build.job_id >>= fun job_name -> let destdir = Fpath.(datadir / job_name / Uuidm.to_string uuid) in Lwt.return (Bos.OS.Path.move staged destdir) | None -> Lwt.return (Bos.OS.Path.delete ~recurse:true Fpath.(staging datadir // staged)) in Lwt.return (Bos.OS.Dir.contents ~rel:true (staging datadir)) >>= fun stageds -> Lwt_result.ok @@ Lwt_list.iter_s (fun staged -> Lwt.map (function | Error (_ as e) -> Log.warn (fun m -> m "Failed cleaning up staged files %a in %a: %a" Fpath.pp staged Fpath.pp Fpath.(staging datadir // staged) pp_error e) | Ok () -> ()) (cleanup_staged staged)) stageds let save path data = let open Lwt.Infix in Lwt.catch (fun () -> Lwt_io.open_file ~mode:Lwt_io.Output (Fpath.to_string path) >>= fun oc -> Lwt_io.write oc data >>= fun () -> Lwt_io.close oc |> Lwt_result.ok) (function | Unix.Unix_error (e, _, _) -> Lwt_result.fail (`Msg (Unix.error_message e)) | e -> Lwt.fail e) let save_artifacts staging artifacts = List.fold_left (fun r (file, data) -> r >>= fun () -> let (`Hex sha256) = Hex.of_cstruct file.Builder_db.sha256 in let destpath = Fpath.(staging / sha256) in save destpath data) (Lwt_result.return ()) artifacts let commit_files datadir staging_dir job_name uuid artifacts = (* First we move the artifacts *) List.fold_left (fun r artifact -> r >>= fun () -> let (`Hex sha256) = Hex.of_cstruct artifact.Builder_db.sha256 in let src = Fpath.(staging_dir / sha256) in let dest = Fpath.(datadir // artifact_path artifact) in Lwt.return (Bos.OS.Dir.create (Fpath.parent dest)) >>= fun _created -> Lwt.return (Bos.OS.Path.move ~force:true src dest)) (Lwt_result.return ()) artifacts >>= fun () -> (* Now the staging dir only contains script & console *) let job_dir = Fpath.(datadir / job_name) in let dest = Fpath.(job_dir / Uuidm.to_string uuid) in Lwt.return (Bos.OS.Dir.create job_dir) >>= fun _ -> Lwt.return (Bos.OS.Path.move staging_dir dest) let infer_section_and_synopsis artifacts = let infer_synopsis_and_descr switch root = match OpamPackage.Name.Map.find_opt root.OpamPackage.name switch.OpamFile.SwitchExport.overlays with | None -> None, None | Some opam -> OpamFile.OPAM.synopsis opam, OpamFile.OPAM.descr_body opam in let infer_section switch root = let root_pkg = root.OpamPackage.name in let is_unikernel = (* since mirage 4.2.0, the x-mirage-opam-lock-location is emitted *) Option.value ~default:false (Option.map (fun opam -> Option.is_some (OpamFile.OPAM.extended opam "x-mirage-opam-lock-location" Fun.id)) (OpamPackage.Name.Map.find_opt root_pkg switch.OpamFile.SwitchExport.overlays)) in let root_pkg_name = OpamPackage.Name.to_string root_pkg in if is_unikernel || Astring.String.is_prefix ~affix:"mirage-unikernel-" root_pkg_name then let metrics_influx = let influx = OpamPackage.Name.of_string "metrics-influx" in OpamPackage.Set.exists (fun p -> OpamPackage.Name.equal p.OpamPackage.name influx) switch.OpamFile.SwitchExport.selections.OpamTypes.sel_installed in let mirage_monitoring = let monitoring = OpamPackage.Name.of_string "mirage-monitoring" in match OpamPackage.Name.Map.find_opt root_pkg switch.OpamFile.SwitchExport.overlays with | None -> false | Some opam -> let depends = OpamFile.OPAM.depends opam in OpamFormula.fold_left (fun acc (n', _) -> acc || OpamPackage.Name.equal n' monitoring) false depends in if metrics_influx || mirage_monitoring then "Unikernels (with metrics reported to Influx)" else "Unikernels" else "Packages" in match List.find_opt (fun (p, _) -> String.equal (Fpath.basename p) "opam-switch") artifacts with | None -> None, (None, None) | Some (_, data) -> try let switch = OpamFile.SwitchExport.read_from_string data in let root = switch.OpamFile.SwitchExport.selections.OpamTypes.sel_roots in assert (OpamPackage.Set.cardinal root = 1); let root = OpamPackage.Set.choose root in Some (infer_section switch root), infer_synopsis_and_descr switch root with _ -> None, (None, None) let compute_input_id artifacts = let get_hash filename = match List.find_opt (fun b -> Fpath.equal b.Builder_db.filepath filename) artifacts with | None -> None | Some x -> Some x.sha256 in match get_hash (Fpath.v "opam-switch"), get_hash (Fpath.v "build-environment"), get_hash (Fpath.v "system-packages") with | Some a, Some b, Some c -> Some (Mirage_crypto.Hash.SHA256.digest (Cstruct.concat [a;b;c])) | _ -> None let save_console_and_script staging_dir job_name uuid console script = let out name = Fpath.(v job_name / Uuidm.to_string uuid / name + "txt") in let out_staging name = Fpath.(staging_dir / name + "txt") in let console_to_string console = List.rev_map (fun (delta, data) -> Printf.sprintf "%.3fs:%s\n" (Duration.to_f (Int64.of_int delta)) data) console |> String.concat "" in save (out_staging "script") script >>= fun () -> save (out_staging "console") (console_to_string console) >|= fun () -> (out "console", out "script") let prepare_staging staging_dir = Lwt.return (Bos.OS.Dir.create staging_dir) >>= fun created -> if not created then Lwt_result.fail (`Msg "build directory already exists") else Lwt_result.return () (* saving: - for each artifact compute its sha256 checksum -- calling Lwt.pause in between - lookup artifact sha256 in the database and filter them out of the list: not_in_db - mkdir -p _staging/uuid/ - save console & script to _staging/uuid/ - save each artifact in not_in_db as _staging/uuid/sha256 committing: - for each artifact mv _staging/uuid/sha256 _artifacts/sha256 (or _artifacts/prefix(sha256)/sha256 where prefix(sha256) is the first two hex digits in sha256) - now _staging/uuid only contains console & script so we mv _staging/uuid _staging/job/uuid potential issues: - race condition in uploading same artifact: * if the artifact already exists in the database and thus filesystem then nothing is done * if the artifact is added to the database and/or filesystem we atomically overwrite it - input_id depends on a sort order? *) let add_build ~datadir ~cachedir ~configdir user_id ((job : Builder.script_job), uuid, console, start, finish, result, raw_artifacts) (module Db : CONN) = let open Builder_db in let job_name = job.Builder.name in let staging_dir = Fpath.(staging datadir / Uuidm.to_string uuid) in let or_cleanup x = Lwt_result.map_error (fun e -> Bos.OS.Dir.delete ~recurse:true staging_dir |> Result.iter_error (fun e -> Log.err (fun m -> m "Failed to remove staging dir %a: %a" Fpath.pp staging_dir pp_error e)); e) x in let not_interesting p = String.equal (Fpath.basename p) "README.md" || String.equal (Fpath.get_ext p) ".build-hashes" in begin List.fold_left (fun r (filepath, data) -> r >>= fun acc -> if not_interesting filepath then Lwt_result.return acc else let sha256 = Mirage_crypto.Hash.SHA256.digest (Cstruct.of_string data) and size = String.length data in Lwt_result.ok (Lwt.pause ()) >|= fun () -> ({ filepath; sha256; size }, data) :: acc) (Lwt_result.return []) raw_artifacts end >>= fun artifacts -> or_cleanup (prepare_staging staging_dir) >>= fun () -> or_cleanup (save_console_and_script staging_dir job_name uuid console job.Builder.script) >>= fun (console, script) -> List.fold_left (fun r ((f, _) as artifact) -> r >>= fun acc -> Db.find Builder_db.Build_artifact.exists f.sha256 >|= fun exists -> if exists then acc else artifact :: acc) (Lwt_result.return []) artifacts >>= fun artifacts_to_save -> or_cleanup (save_artifacts staging_dir artifacts_to_save) >>= fun () -> let artifacts = List.map fst artifacts in let r = Db.start () >>= fun () -> Db.exec Job.try_add job_name >>= fun () -> Db.find_opt Job.get_id_by_name job_name >>= fun job_id -> Lwt.return (Option.to_result ~none:(`Msg "No such job id") job_id) >>= fun job_id -> let section_tag = "section" in Db.exec Tag.try_add section_tag >>= fun () -> Db.find Tag.get_id_by_name section_tag >>= fun section_id -> let synopsis_tag = "synopsis" in Db.exec Tag.try_add synopsis_tag >>= fun () -> Db.find Tag.get_id_by_name synopsis_tag >>= fun synopsis_id -> let descr_tag = "description" in Db.exec Tag.try_add descr_tag >>= fun () -> Db.find Tag.get_id_by_name descr_tag >>= fun descr_id -> let readme_tag = "readme.md" in Db.exec Tag.try_add readme_tag >>= fun () -> Db.find Tag.get_id_by_name readme_tag >>= fun readme_id -> let input_id = compute_input_id artifacts in let platform = job.Builder.platform in Db.exec Build.add { Build.uuid; start; finish; result; console; script; platform; main_binary = None; input_id; user_id; job_id } >>= fun () -> Db.find last_insert_rowid () >>= fun id -> let sec_syn = infer_section_and_synopsis raw_artifacts in let add_or_update tag_id tag_value = Db.find_opt Job_tag.get_value (tag_id, job_id) >>= function | None -> Db.exec Job_tag.add (tag_id, tag_value, job_id) | Some _ -> Db.exec Job_tag.update (tag_id, tag_value, job_id) in (match fst sec_syn with | None -> Lwt_result.return () | Some section_v -> add_or_update section_id section_v) >>= fun () -> (match snd sec_syn with | None, _-> Lwt_result.return () | Some synopsis_v, _ -> add_or_update synopsis_id synopsis_v) >>= fun () -> (match snd sec_syn with | _, None -> Lwt_result.return () | _, Some descr_v -> add_or_update descr_id descr_v) >>= fun () -> (let readme = List.find_opt (fun (p, _) -> Fpath.(equal (v "README.md") p)) raw_artifacts in let readme_anywhere = List.find_opt (fun (p, _) -> String.equal "README.md" (Fpath.basename p)) raw_artifacts in match readme, readme_anywhere with | None, None -> Lwt_result.return () | Some (_, data), _ | None, Some (_, data) -> add_or_update readme_id data) >>= fun () -> (match List.partition (fun p -> Fpath.(is_prefix (v "bin/") p.filepath)) artifacts with | [ main_binary ], other_artifacts -> Db.exec Build_artifact.add (main_binary, id) >>= fun () -> Db.find Builder_db.last_insert_rowid () >>= fun main_binary_id -> Db.exec Build.set_main_binary (id, main_binary_id) >|= fun () -> Some main_binary, other_artifacts | [], _ -> Log.debug (fun m -> m "Zero binaries for build %a" Uuidm.pp uuid); Lwt_result.return (None, artifacts) | xs, _ -> Log.warn (fun m -> m "Multiple binaries for build %a: %a" Uuidm.pp uuid Fmt.(list ~sep:(any ",") Fpath.pp) (List.map (fun a -> a.filepath) xs)); Lwt_result.return (None, artifacts)) >>= fun (main_binary, remaining_artifacts_to_add) -> List.fold_left (fun r file -> r >>= fun () -> Db.exec Build_artifact.add (file, id)) (Lwt_result.return ()) remaining_artifacts_to_add >>= fun () -> Db.commit () >>= fun () -> commit_files datadir staging_dir job_name uuid artifacts >|= fun () -> main_binary in Lwt_result.bind_lwt_error (or_cleanup r) (fun e -> Db.rollback () |> Lwt.map (fun r -> Result.iter_error (fun e' -> Log.err (fun m -> m "Failed rollback: %a" Caqti_error.pp e')) r; e)) >>= function | None -> Lwt.return (Ok ()) | Some main_binary -> let time = let (y, m, d), ((hh, mm, ss), _) = Ptime.to_date_time start in Printf.sprintf "%04d%02d%02d%02d%02d%02d" y m d hh mm ss and uuid = Uuidm.to_string uuid and job = job.name and platform = job.platform and `Hex sha256 = Hex.of_cstruct main_binary.sha256 in let fp_str p = Fpath.(to_string (datadir // p)) in let args = String.concat " " (List.map (fun s -> "\"" ^ String.escaped s ^ "\"") [ "--build-time=" ^ time ; "--sha256=" ^ sha256 ; "--job=" ^ job ; "--uuid=" ^ uuid ; "--platform=" ^ platform ; "--cache-dir=" ^ Fpath.to_string cachedir ; "--data-dir=" ^ Fpath.to_string datadir ; fp_str Fpath.(datadir // artifact_path main_binary) ]) in Log.debug (fun m -> m "executing hooks with %s" args); let dir = Fpath.(configdir / "upload-hooks") in (try Lwt.return (Ok (Some (Unix.opendir (Fpath.to_string dir)))) with Unix.Unix_error _ -> Lwt.return (Ok None)) >>= function | None -> Lwt.return (Ok ()) | Some dh -> try let is_executable file = let st = Unix.stat (Fpath.to_string file) in st.Unix.st_perm land 0o111 = 0o111 && st.Unix.st_kind = Unix.S_REG in let rec go () = let next_file = Unix.readdir dh in let file = Fpath.(dir / next_file) in if is_executable file && Fpath.has_ext ".sh" file then ignore (Sys.command (Fpath.to_string file ^ " " ^ args ^ " &")); go () in go () with | End_of_file -> Unix.closedir dh; Lwt.return (Ok ())