open Rresult.R.Infix let save_data outputdir (filepath, data) = let localpath = Fpath.(outputdir // filepath) in (* FIXME: return an error?! *) let () = match Bos.OS.File.exists localpath with | Ok false -> Logs.warn (fun m -> m "artifact file %a does not exist in %a" Fpath.pp filepath Fpath.pp outputdir) | Error (`Msg e) -> Logs.warn (fun m -> m "artifact file error %a: %s" Fpath.pp localpath e) | Ok true -> () in (filepath, localpath, data) let get_by_uuid uuid (module Db : Caqti_blocking.CONNECTION) = Uuidm.of_string uuid |> Option.to_result ~none:"bad uuid" >>= fun uuid -> Db.find_opt Builder_db.Build.get_by_uuid uuid |> Result.map_error (fun e -> Fmt.strf "Error getting build %a: %a" Uuidm.pp uuid Caqti_error.pp e) let db_add_build (job, uuid, console, start, finish, result, artifacts) (input_files : (Fpath.t * Fpath.t * string) list) (module Db : Caqti_blocking.CONNECTION) = let open Builder_db in let job_name = job.Builder.name in Db.exec Job.try_add job_name >>= fun () -> Db.find Job.get_id_by_name job_name >>= fun job_id -> Db.exec Build.add { Build.uuid; start; finish; result; console; script = job.Builder.script; job_id } >>= fun () -> Db.find last_insert_rowid () >>= fun id -> List.fold_left (fun r (filepath, localpath, data) -> r >>= fun () -> let sha256 = Mirage_crypto.Hash.SHA256.digest (Cstruct.of_string data) in Db.exec Build_artifact.add ({ filepath; localpath; sha256 }, id)) (Ok ()) artifacts >>= fun () -> List.fold_left (fun r (filepath, localpath, data) -> r >>= fun () -> let sha256 = Mirage_crypto.Hash.SHA256.digest (Cstruct.of_string data) in Db.exec Build_file.add ({ filepath; localpath; sha256 }, id)) (Ok ()) input_files let add_build conn builddir = let f = Fpath.(builddir / "full") in let outputdir = Fpath.(builddir / "output") in let inputdir = Fpath.(builddir / "input") in let uuid = Fpath.basename builddir in match get_by_uuid uuid conn with | Error e -> Logs.warn (fun m -> m "%s" e) | Ok (Some _) -> Logs.debug (fun m -> m "Skipping %a, already in database" Fpath.pp builddir) | Ok None -> Logs.debug (fun m -> m "Adding build %a" Fpath.pp builddir); match Bos.OS.File.read f with | Error (`Msg e) -> Logs.warn (fun m -> m "Error getting build %a: %s" Fpath.pp builddir e) | Ok contents -> match Builder.Asn.exec_of_cs (Cstruct.of_string contents) with | Error (`Msg e) -> Logs.warn (fun m -> m "Error parsing build file %a: %s" Fpath.pp f e) | Ok (job, uuid, console, start, finish, result, data) -> let data = List.map (save_data outputdir) data in let input_files = List.map (save_data inputdir) job.Builder.files in match db_add_build (job, uuid, console, start, finish, result, data) input_files conn with | Error e -> Logs.err (fun m -> m "Error inserting build %a: %a" Fpath.pp builddir Caqti_error.pp e) | Ok () -> () let add_job conn jobdir = Logs.debug (fun m -> m "Adding job %a" Fpath.pp jobdir); match Bos.OS.Dir.contents jobdir with | Error (`Msg e) -> Logs.warn (fun m -> m "Error getting job %s: %s\n" (Fpath.basename jobdir) e) | Ok builds -> List.iter (add_build conn) builds let add_jobs conn datadir = Bos.OS.Dir.contents datadir >>| List.filter (fun f -> not Fpath.(equal (v "state") f)) >>| List.iter (add_job conn) let add () dbpath datadir = let datadir = Fpath.v datadir in Logs.debug (fun m -> m "Data dir: %a" Fpath.pp datadir); let conn = match Caqti_blocking.connect (Uri.make ~scheme:"sqlite3" ~path:dbpath ~query:["create", ["false"]] ()) with | Error e -> Logs.err (fun m -> m "Error connecting to database: %a" Caqti_error.pp e); exit 1 | Ok conn -> conn in match add_jobs conn datadir with | Ok () -> () | Error (`Msg e) -> Logs.err (fun m -> m "Error getting jobs: %s\n" e); exit 2 let do_migrate dbpath = Caqti_blocking.connect (Uri.make ~scheme:"sqlite3" ~path:dbpath ()) >>= fun (module Db : Caqti_blocking.CONNECTION) -> List.fold_left (fun r migrate -> r >>= fun () -> Logs.debug (fun m -> m "Executing migration query: %a" Caqti_request.pp migrate); Db.exec migrate ()) (Ok ()) Builder_db.migrate let migrate () dbpath = match do_migrate dbpath with | Ok () -> () | Error e -> Format.eprintf "Database error: %a" Caqti_error.pp e; exit 1 let help man_format cmds = function | None -> `Help (man_format, None) | Some cmd -> if List.mem cmd cmds then `Help (man_format, Some cmd) else `Error (true, "Unknown command: " ^ cmd) let dbpath = let doc = "sqlite3 database path" in Cmdliner.Arg.(value & opt non_dir_file "builder.sqlite3" & info ~doc ["dbpath"]) let dbpath_new = let doc = "sqlite3 database path" in Cmdliner.Arg.(value & opt string "builder.sqlite3" & info ~doc ["dbpath"]) let datadir = let doc = Cmdliner.Arg.info ~doc:"builder data dir" ["datadir"] in Cmdliner.Arg.(value & opt dir "/var/db/builder/" doc) let setup_log = let setup_log level = Logs.set_level level; Logs.set_reporter (Logs_fmt.reporter ~dst:Format.std_formatter ()); Logs.debug (fun m -> m "Set log level %s" (Logs.level_to_string level)) in Cmdliner.Term.(const setup_log $ Logs_cli.level ()) let migrate_cmd = let doc = "create database and add tables" in Cmdliner.Term.(pure migrate $ setup_log $ dbpath_new), Cmdliner.Term.info ~doc "migrate" let add_cmd = let doc = "populates database with builder data" in let man = [ `S "DESCRIPTION"; `P "Scrape builder data directory information and insert into builder-web database."; `P "It assumes the `full' files are stored in a directory hierarchy of the following shape:"; `Pre "/path/to/datadir/JOB-NAME/BUILD-UUID/full"; `P "Before parsing, The UUID in the filesystem is looked up in the database \ to see if already exists.\ It is assumed the UUIDs correspond."; ] in (Cmdliner.Term.(pure add $ setup_log $ dbpath $ datadir), Cmdliner.Term.info ~doc ~man "add") let help_cmd = let topic = let doc = "Command to get help on" in Cmdliner.Arg.(value & pos 0 (some string) None & info ~doc ~docv:"COMMAND" []) in let doc = "Builder database help" in Cmdliner.Term.(ret (const help $ man_format $ choice_names $ topic)), Cmdliner.Term.info ~doc "help" let default_cmd = let doc = "Builder database command" in Cmdliner.Term.(ret (const help $ man_format $ choice_names $ const None)), Cmdliner.Term.info ~doc "builder-db" let () = Cmdliner.Term.eval_choice default_cmd [help_cmd; add_cmd; migrate_cmd] |> Cmdliner.Term.exit