builder-web/bin/builder_db.ml

197 lines
6.9 KiB
OCaml
Raw Normal View History

open Rresult.R.Infix
let save_data outputdir (filepath, data) =
let localpath = Fpath.(outputdir // filepath) in
(* FIXME: return an error?! *)
let () =
match Bos.OS.File.exists localpath with
| Ok false ->
Logs.warn (fun m -> m "artifact file %a does not exist in %a"
Fpath.pp filepath Fpath.pp outputdir)
| Error (`Msg e) ->
Logs.warn (fun m -> m "artifact file error %a: %s"
Fpath.pp localpath e)
| Ok true -> ()
in
(filepath, localpath, data)
let get_by_uuid uuid (module Db : Caqti_blocking.CONNECTION) =
Uuidm.of_string uuid |> Option.to_result ~none:"bad uuid" >>= fun uuid ->
Db.find_opt Builder_db.Build.get_by_uuid uuid
|> Result.map_error (fun e ->
Fmt.strf "Error getting build %a: %a" Uuidm.pp uuid Caqti_error.pp e)
let db_add_build (job, uuid, console, start, finish, result, artifacts)
(input_files : (Fpath.t * Fpath.t * string) list)
(module Db : Caqti_blocking.CONNECTION) =
let open Builder_db in
let job_name = job.Builder.name in
Db.exec Job.try_add job_name >>= fun () ->
Db.find Job.get_id_by_name job_name >>= fun job_id ->
Db.exec Build.add { Build.uuid; start; finish; result; console;
script = job.Builder.script; job_id } >>= fun () ->
Db.find last_insert_rowid () >>= fun id ->
List.fold_left
(fun r (filepath, localpath, data) ->
r >>= fun () ->
let sha256 = Mirage_crypto.Hash.SHA256.digest (Cstruct.of_string data) in
Db.exec Build_artifact.add ({ filepath; localpath; sha256 }, id))
(Ok ())
artifacts >>= fun () ->
List.fold_left
(fun r (filepath, localpath, data) ->
r >>= fun () ->
let sha256 = Mirage_crypto.Hash.SHA256.digest (Cstruct.of_string data) in
Db.exec Build_file.add ({ filepath; localpath; sha256 }, id))
(Ok ())
input_files
let add_build conn builddir =
let f = Fpath.(builddir / "full") in
let outputdir = Fpath.(builddir / "output") in
let inputdir = Fpath.(builddir / "input") in
let uuid = Fpath.basename builddir in
match get_by_uuid uuid conn with
| Error e -> Logs.warn (fun m -> m "%s" e)
| Ok (Some _) -> Logs.debug (fun m -> m "Skipping %a, already in database" Fpath.pp builddir)
| Ok None ->
Logs.debug (fun m -> m "Adding build %a" Fpath.pp builddir);
match Bos.OS.File.read f with
| Error (`Msg e) ->
Logs.warn (fun m -> m "Error getting build %a: %s"
Fpath.pp builddir e)
| Ok contents ->
match Builder.Asn.exec_of_cs (Cstruct.of_string contents) with
| Error (`Msg e) ->
Logs.warn (fun m -> m "Error parsing build file %a: %s"
Fpath.pp f e)
| Ok (job, uuid, console, start, finish, result, data) ->
let data = List.map (save_data outputdir) data in
let input_files = List.map (save_data inputdir) job.Builder.files in
match db_add_build (job, uuid, console, start, finish, result, data) input_files conn with
| Error e ->
Logs.err (fun m -> m "Error inserting build %a: %a"
Fpath.pp builddir Caqti_error.pp e)
| Ok () -> ()
let add_job conn jobdir =
Logs.debug (fun m -> m "Adding job %a" Fpath.pp jobdir);
match Bos.OS.Dir.contents jobdir with
| Error (`Msg e) ->
Logs.warn (fun m ->
m "Error getting job %s: %s\n" (Fpath.basename jobdir) e)
| Ok builds ->
List.iter (add_build conn) builds
let add_jobs conn datadir =
Bos.OS.Dir.contents datadir >>|
List.filter (fun f -> not Fpath.(equal (v "state") f)) >>|
List.iter (add_job conn)
let add () dbpath datadir =
let datadir = Fpath.v datadir in
Logs.debug (fun m -> m "Data dir: %a" Fpath.pp datadir);
let conn =
match Caqti_blocking.connect (Uri.make ~scheme:"sqlite3" ~path:dbpath ~query:["create", ["false"]] ()) with
| Error e ->
Logs.err (fun m -> m "Error connecting to database: %a" Caqti_error.pp e);
exit 1
| Ok conn ->
conn
in
match add_jobs conn datadir with
| Ok () -> ()
| Error (`Msg e) ->
Logs.err (fun m -> m "Error getting jobs: %s\n" e);
exit 2
let do_migrate dbpath =
Caqti_blocking.connect (Uri.make ~scheme:"sqlite3" ~path:dbpath ())
>>= fun (module Db : Caqti_blocking.CONNECTION) ->
List.fold_left
(fun r migrate ->
r >>= fun () ->
Logs.debug (fun m -> m "Executing migration query: %a" Caqti_request.pp migrate);
Db.exec migrate ())
(Ok ())
Builder_db.migrate
let migrate () dbpath =
match do_migrate dbpath with
| Ok () -> ()
| Error e ->
Format.eprintf "Database error: %a" Caqti_error.pp e;
exit 1
let help man_format cmds = function
| None -> `Help (man_format, None)
| Some cmd ->
if List.mem cmd cmds
then `Help (man_format, Some cmd)
else `Error (true, "Unknown command: " ^ cmd)
let dbpath =
let doc = "sqlite3 database path" in
Cmdliner.Arg.(value &
opt non_dir_file "builder.sqlite3" &
info ~doc ["dbpath"])
let dbpath_new =
let doc = "sqlite3 database path" in
Cmdliner.Arg.(value &
opt string "builder.sqlite3" &
info ~doc ["dbpath"])
let datadir =
let doc = Cmdliner.Arg.info ~doc:"builder data dir" ["datadir"] in
Cmdliner.Arg.(value &
opt dir "/var/db/builder/" doc)
let setup_log =
let setup_log level =
Logs.set_level level;
Logs.set_reporter (Logs_fmt.reporter ~dst:Format.std_formatter ());
Logs.debug (fun m -> m "Set log level %s" (Logs.level_to_string level))
in
Cmdliner.Term.(const setup_log $ Logs_cli.level ())
let migrate_cmd =
let doc = "create database and add tables" in
Cmdliner.Term.(pure migrate $ setup_log $ dbpath_new),
Cmdliner.Term.info ~doc "migrate"
let add_cmd =
let doc = "populates database with builder data" in
let man =
[ `S "DESCRIPTION";
`P "Scrape builder data directory information and insert into builder-web database.";
`P "It assumes the `full' files are stored in a directory hierarchy of the following shape:";
`Pre "/path/to/datadir/JOB-NAME/BUILD-UUID/full";
`P "Before parsing, The UUID in the filesystem is looked up in the database \
to see if already exists.\
It is assumed the UUIDs correspond.";
]
in
(Cmdliner.Term.(pure add $ setup_log $ dbpath $ datadir),
Cmdliner.Term.info ~doc ~man "add")
let help_cmd =
let topic =
let doc = "Command to get help on" in
Cmdliner.Arg.(value & pos 0 (some string) None & info ~doc ~docv:"COMMAND" [])
in
let doc = "Builder database help" in
Cmdliner.Term.(ret (const help $ man_format $ choice_names $ topic)),
Cmdliner.Term.info ~doc "help"
let default_cmd =
let doc = "Builder database command" in
Cmdliner.Term.(ret (const help $ man_format $ choice_names $ const None)),
Cmdliner.Term.info ~doc "builder-db"
let () =
Cmdliner.Term.eval_choice
default_cmd
[help_cmd; add_cmd; migrate_cmd]
|> Cmdliner.Term.exit