From a870894475edaf4a9ad2667101f47e2009f32e71 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Reynir=20Bj=C3=B6rnsson?= Date: Tue, 14 Jan 2025 17:11:35 +0100 Subject: [PATCH 1/4] Add exec endpoint --- lib/builder_web.ml | 12 ++++++++++++ lib/model.ml | 40 ++++++++++++++++++++++++++++++++++++++++ lib/model.mli | 3 +++ 3 files changed, 55 insertions(+) diff --git a/lib/builder_web.ml b/lib/builder_web.ml index de2a7c1..92ff5b2 100644 --- a/lib/builder_web.ml +++ b/lib/builder_web.ml @@ -516,6 +516,17 @@ let routes ~datadir ~cachedir ~configdir ~expired_jobs = |> Lwt_result.ok in + let job_build_full req = + let _job_name = Dream.param req "job" + and build = Dream.param req "build" in + get_uuid build >>= fun uuid -> + Dream.sql req (Model.exec_of_build datadir uuid) + |> if_error "Error getting build" >>= fun exec -> + ignore exec; + Dream.respond ~status:`Not_Found "" + |> Lwt_result.ok + in + let upload req = let* body = Dream.body req in Builder.Asn.exec_of_str body |> Lwt.return @@ -708,6 +719,7 @@ let routes ~datadir ~cachedir ~configdir ~expired_jobs = `Get, "/job/:job/build/:build/script", (w (job_build_static_file `Script)); `Get, "/job/:job/build/:build/console", (w (job_build_static_file `Console)); `Get, "/job/:job/build/:build/all.tar.gz", (w job_build_targz); + `Get, "/job/:job/build/:build/exec", (w job_build_full); `Get, "/failed-builds", (w failed_builds); `Get, "/all-builds", (w (builds ~all:true)); `Get, "/hash", (w hash); diff --git a/lib/model.ml b/lib/model.ml index 7cd7d9f..e8c6f15 100644 --- a/lib/model.ml +++ b/lib/model.ml @@ -523,3 +523,43 @@ let add_build | End_of_file -> Unix.closedir dh; Lwt.return (Ok ()) + +let console_of_string data = + let lines = String.split_on_char '\n' data in + (* remove last empty line *) + let lines = + match List.rev lines with + | "" :: lines -> List.rev lines + | _ -> lines + in + List.map (fun line -> + match String.split_on_char ':' line with + | ts :: tail -> + let delta = float_of_string (String.sub ts 0 (String.length ts - 1)) in + Int64.to_int (Duration.of_f delta), String.concat ":" tail + | _ -> assert false) + lines + +let exec_of_build datadir uuid (module Db : CONN) = + let open Builder_db in + Db.find_opt Build.get_by_uuid uuid >>= not_found >>= fun (build_id, build) -> + let { Builder_db.Build.start; finish; result; + job_id; console; script; platform; _ } = + build + in + Db.find Builder_db.Job.get job_id >>= fun job_name -> + read_file datadir script >>= fun script -> + let job = { Builder.name = job_name; platform; script } in + read_file datadir console >>= fun console -> + let out = console_of_string console in + Db.collect_list Builder_db.Build_artifact.get_all_by_build build_id >>= fun artifacts -> + Lwt_list.fold_left_s (fun acc (_id, ({ filepath; _ } as file)) -> + match acc with + | Error _ as e -> Lwt.return e + | Ok acc -> + build_artifact_data datadir file >>= fun data -> + Lwt.return (Ok ((filepath, data) :: acc))) + (Ok []) artifacts >>= fun data -> + let exec = (job, uuid, out, start, finish, result, data) in + let data = Builder.Asn.exec_to_str exec in + Lwt.return (Ok data) diff --git a/lib/model.mli b/lib/model.mli index b59e19c..555ad7f 100644 --- a/lib/model.mli +++ b/lib/model.mli @@ -104,3 +104,6 @@ val add_build : Builder.execution_result * (Fpath.t * string) list) -> Caqti_lwt.connection -> (unit, [> Caqti_error.call_or_retrieve | `Msg of string ]) result Lwt.t + +val exec_of_build : Fpath.t -> Uuidm.t -> Caqti_lwt.connection -> + (string, [> Caqti_error.call_or_retrieve | `Not_found | `File_error of Fpath.t ]) result Lwt.t From 76ab74134471b813b7b1ef7bcc5e4bc7f0ded30c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Reynir=20Bj=C3=B6rnsson?= Date: Wed, 15 Jan 2025 09:52:48 +0100 Subject: [PATCH 2/4] Actually return the builder exec data --- lib/builder_web.ml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/lib/builder_web.ml b/lib/builder_web.ml index 92ff5b2..0a6ec41 100644 --- a/lib/builder_web.ml +++ b/lib/builder_web.ml @@ -522,8 +522,7 @@ let routes ~datadir ~cachedir ~configdir ~expired_jobs = get_uuid build >>= fun uuid -> Dream.sql req (Model.exec_of_build datadir uuid) |> if_error "Error getting build" >>= fun exec -> - ignore exec; - Dream.respond ~status:`Not_Found "" + Dream.respond ~headers:["Content-Type", "application/octet-stream"] exec |> Lwt_result.ok in From 3c7c21afcd3d47afcf649de49930fd649bd2215a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Reynir=20Bj=C3=B6rnsson?= Date: Wed, 15 Jan 2025 13:36:31 +0100 Subject: [PATCH 3/4] Improve console_of_string --- bin/builder_db_app.ml | 24 ++++++++++++++---------- lib/model.ml | 24 ++++++++++++++---------- 2 files changed, 28 insertions(+), 20 deletions(-) diff --git a/bin/builder_db_app.ml b/bin/builder_db_app.ml index 0ade5e4..e6f4283 100644 --- a/bin/builder_db_app.ml +++ b/bin/builder_db_app.ml @@ -759,20 +759,24 @@ module Asn = struct let console_of_cs, console_to_cs = projections_of console end +(* NOTE: this function is duplicatedi in lib/model.ml *) let console_of_string data = - let lines = String.split_on_char '\n' data in (* remove last empty line *) - let lines = - match List.rev lines with - | "" :: lines -> List.rev lines - | _ -> lines + let data = + if String.ends_with ~suffix:"\n" data then + String.sub data 0 (String.length data - 2) + else data in + let lines = String.split_on_char '\n' data in List.map (fun line -> - match String.split_on_char ':' line with - | ts :: tail -> - let delta = float_of_string (String.sub ts 0 (String.length ts - 1)) in - Int64.to_int (Duration.of_f delta), String.concat ":" tail - | _ -> assert false) + match String.index line ':' with + | i -> + (* the timestamp is of the form "%fs", e.g. 0.867s; so chop off the 's' *) + let delta = float_of_string (String.sub line 0 (i - 1)) in + let delta = Int64.to_int (Duration.of_f delta) in + let line = String.sub line i (String.length line - i) in + delta, line + | exception Not_found -> assert false) lines let extract_full () datadir dest uuid = diff --git a/lib/model.ml b/lib/model.ml index e8c6f15..b911e89 100644 --- a/lib/model.ml +++ b/lib/model.ml @@ -524,20 +524,24 @@ let add_build Unix.closedir dh; Lwt.return (Ok ()) +(* NOTE: this function is duplicatedi in bin/builder_db_app.ml *) let console_of_string data = - let lines = String.split_on_char '\n' data in (* remove last empty line *) - let lines = - match List.rev lines with - | "" :: lines -> List.rev lines - | _ -> lines + let data = + if String.ends_with ~suffix:"\n" data then + String.sub data 0 (String.length data - 2) + else data in + let lines = String.split_on_char '\n' data in List.map (fun line -> - match String.split_on_char ':' line with - | ts :: tail -> - let delta = float_of_string (String.sub ts 0 (String.length ts - 1)) in - Int64.to_int (Duration.of_f delta), String.concat ":" tail - | _ -> assert false) + match String.index line ':' with + | i -> + (* the timestamp is of the form "%fs", e.g. 0.867s; so chop off the 's' *) + let delta = float_of_string (String.sub line 0 (i - 1)) in + let delta = Int64.to_int (Duration.of_f delta) in + let line = String.sub line i (String.length line - i) in + delta, line + | exception Not_found -> assert false) lines let exec_of_build datadir uuid (module Db : CONN) = From b2f153396b83b497d6aa5edafe5b88ad54b2d237 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Reynir=20Bj=C3=B6rnsson?= Date: Wed, 15 Jan 2025 13:43:18 +0100 Subject: [PATCH 4/4] console_of_string: be a bit more robust If a line somehow has an unexpected shape we log a warning and omit it. --- bin/builder_db_app.ml | 16 +++++++--------- lib/model.ml | 16 +++++++--------- 2 files changed, 14 insertions(+), 18 deletions(-) diff --git a/bin/builder_db_app.ml b/bin/builder_db_app.ml index e6f4283..d11e4d0 100644 --- a/bin/builder_db_app.ml +++ b/bin/builder_db_app.ml @@ -761,22 +761,20 @@ end (* NOTE: this function is duplicatedi in lib/model.ml *) let console_of_string data = - (* remove last empty line *) - let data = - if String.ends_with ~suffix:"\n" data then - String.sub data 0 (String.length data - 2) - else data - in let lines = String.split_on_char '\n' data in - List.map (fun line -> + List.filter_map (fun line -> match String.index line ':' with + | 0 -> Logs.warn (fun m -> m "console line starting with colon %S" line); None | i -> (* the timestamp is of the form "%fs", e.g. 0.867s; so chop off the 's' *) let delta = float_of_string (String.sub line 0 (i - 1)) in let delta = Int64.to_int (Duration.of_f delta) in let line = String.sub line i (String.length line - i) in - delta, line - | exception Not_found -> assert false) + Some (delta, line) + | exception Not_found -> + if line <> "" then + Logs.warn (fun m -> m "Unexpected console line %S" line); + None) lines let extract_full () datadir dest uuid = diff --git a/lib/model.ml b/lib/model.ml index b911e89..12a121e 100644 --- a/lib/model.ml +++ b/lib/model.ml @@ -526,22 +526,20 @@ let add_build (* NOTE: this function is duplicatedi in bin/builder_db_app.ml *) let console_of_string data = - (* remove last empty line *) - let data = - if String.ends_with ~suffix:"\n" data then - String.sub data 0 (String.length data - 2) - else data - in let lines = String.split_on_char '\n' data in - List.map (fun line -> + List.filter_map (fun line -> match String.index line ':' with + | 0 -> Log.warn (fun m -> m "console line starting with colon %S" line); None | i -> (* the timestamp is of the form "%fs", e.g. 0.867s; so chop off the 's' *) let delta = float_of_string (String.sub line 0 (i - 1)) in let delta = Int64.to_int (Duration.of_f delta) in let line = String.sub line i (String.length line - i) in - delta, line - | exception Not_found -> assert false) + Some (delta, line) + | exception Not_found -> + if line <> "" then + Log.warn (fun m -> m "Unexpected console line %S" line); + None) lines let exec_of_build datadir uuid (module Db : CONN) =