Merge pull request 'Endpoint for builder exec asn.1 data' (!10) from download-full into main

Reviewed-on: #10

bin/builder_db_app.ml

@@ -759,20 +759,22 @@ module Asn = struct
   let console_of_cs, console_to_cs = projections_of console
 end
 
+(* NOTE: this function is duplicatedi in lib/model.ml *)
 let console_of_string data =
   let lines = String.split_on_char '\n' data in
-  (* remove last empty line *)
+  List.filter_map (fun line ->
-  let lines =
+      match String.index line ':' with
-    match List.rev lines with
+      | 0 -> Logs.warn (fun m -> m "console line starting with colon %S" line); None
-    | "" :: lines -> List.rev lines
+      | i ->
-    | _ -> lines
+        (* the timestamp is of the form "%fs", e.g. 0.867s; so chop off the 's' *)
-  in
+        let delta = float_of_string (String.sub line 0 (i - 1)) in
-  List.map (fun line ->
+        let delta = Int64.to_int (Duration.of_f delta) in
-      match String.split_on_char ':' line with
+        let line = String.sub line i (String.length line - i) in
-      | ts :: tail ->
+        Some (delta, line)
-        let delta = float_of_string (String.sub ts 0 (String.length ts - 1)) in
+      | exception Not_found ->
-        Int64.to_int (Duration.of_f delta), String.concat ":" tail
+        if line <> "" then
-      | _ -> assert false)
+          Logs.warn (fun m -> m "Unexpected console line %S" line);
+        None)
     lines
 
 let extract_full () datadir dest uuid =

lib/builder_web.ml

@@ -516,6 +516,16 @@ let routes ~datadir ~cachedir ~configdir ~expired_jobs =
     |> Lwt_result.ok
   in
 
+  let job_build_full req =
+    let _job_name = Dream.param req "job"
+    and build = Dream.param req "build" in
+    get_uuid build >>= fun uuid ->
+    Dream.sql req (Model.exec_of_build datadir uuid)
+    |> if_error "Error getting build" >>= fun exec ->
+    Dream.respond ~headers:["Content-Type", "application/octet-stream"] exec
+    |> Lwt_result.ok
+  in
+
   let upload req =
     let* body = Dream.body req in
     Builder.Asn.exec_of_str body |> Lwt.return
@@ -708,6 +718,7 @@ let routes ~datadir ~cachedir ~configdir ~expired_jobs =
     `Get, "/job/:job/build/:build/script", (w (job_build_static_file `Script));
     `Get, "/job/:job/build/:build/console", (w (job_build_static_file `Console));
     `Get, "/job/:job/build/:build/all.tar.gz", (w job_build_targz);
+    `Get, "/job/:job/build/:build/exec", (w job_build_full);
     `Get, "/failed-builds", (w failed_builds);
     `Get, "/all-builds", (w (builds ~all:true));
     `Get, "/hash", (w hash);

lib/model.ml

@@ -523,3 +523,45 @@ let add_build
       | End_of_file ->
         Unix.closedir dh;
         Lwt.return (Ok ())
+
+(* NOTE: this function is duplicatedi in bin/builder_db_app.ml *)
+let console_of_string data =
+  let lines = String.split_on_char '\n' data in
+  List.filter_map (fun line ->
+      match String.index line ':' with
+      | 0 -> Log.warn (fun m -> m "console line starting with colon %S" line); None
+      | i ->
+        (* the timestamp is of the form "%fs", e.g. 0.867s; so chop off the 's' *)
+        let delta = float_of_string (String.sub line 0 (i - 1)) in
+        let delta = Int64.to_int (Duration.of_f delta) in
+        let line = String.sub line i (String.length line - i) in
+        Some (delta, line)
+      | exception Not_found ->
+        if line <> "" then
+          Log.warn (fun m -> m "Unexpected console line %S" line);
+        None)
+    lines
+
+let exec_of_build datadir uuid (module Db : CONN) =
+  let open Builder_db in
+  Db.find_opt Build.get_by_uuid uuid >>= not_found >>= fun (build_id, build) ->
+  let { Builder_db.Build.start; finish; result;
+        job_id; console; script; platform; _ } =
+    build
+  in
+  Db.find Builder_db.Job.get job_id >>= fun job_name ->
+  read_file datadir script >>= fun script ->
+  let job = { Builder.name = job_name; platform; script } in
+  read_file datadir console >>= fun console ->
+  let out = console_of_string console in
+  Db.collect_list Builder_db.Build_artifact.get_all_by_build build_id >>= fun artifacts ->
+  Lwt_list.fold_left_s (fun acc (_id, ({ filepath; _ } as file)) ->
+      match acc with
+      | Error _ as e -> Lwt.return e
+      | Ok acc ->
+        build_artifact_data datadir file >>= fun data ->
+        Lwt.return (Ok ((filepath, data) :: acc)))
+    (Ok []) artifacts >>= fun data ->
+  let exec = (job, uuid, out, start, finish, result, data) in
+  let data = Builder.Asn.exec_to_str exec in
+  Lwt.return (Ok data)

lib/model.mli

@@ -104,3 +104,6 @@ val add_build :
    Builder.execution_result * (Fpath.t * string) list) ->
   Caqti_lwt.connection ->
   (unit, [> Caqti_error.call_or_retrieve | `Msg of string ]) result Lwt.t
+
+val exec_of_build : Fpath.t -> Uuidm.t -> Caqti_lwt.connection ->
+  (string, [> Caqti_error.call_or_retrieve | `Not_found | `File_error of Fpath.t ]) result Lwt.t