Merge pull request 'Add more json to endpoints, populate-local.sh' (!11) from more-json into main

Reviewed-on: #11

lib/builder_web.ml

@@ -307,6 +307,9 @@ let routes ~datadir ~cachedir ~configdir ~expired_jobs =
     |> if_error "Error getting jobs"
       ~log:(fun e -> Log.warn (fun m -> m "Error getting jobs: %a" pp_error e))
     >>= fun jobs ->
+    if is_accept_json req then
+      Views.Builds.make_json ~all jobs |> Yojson.Basic.to_string |> Dream.json |> Lwt_result.ok
+    else
       Views.Builds.make ~all jobs |> string_of_html |> Dream.html |> Lwt_result.ok
   in
 
@@ -319,6 +322,11 @@ let routes ~datadir ~cachedir ~configdir ~expired_jobs =
     |> if_error "Error getting job"
       ~log:(fun e -> Log.warn (fun m -> m "Error getting job: %a" pp_error e))
     >>= fun (readme, builds) ->
+    if is_accept_json req then
+      Views.Job.make_json ~failed:false ~job_name ~platform ~readme builds
+      |> Yojson.Basic.to_string
+      |> Dream.json |> Lwt_result.ok
+    else
       Views.Job.make ~failed:false ~job_name ~platform ~readme builds
       |> string_of_html |> Dream.html |> Lwt_result.ok
   in
@@ -332,6 +340,11 @@ let routes ~datadir ~cachedir ~configdir ~expired_jobs =
     |> if_error "Error getting job"
       ~log:(fun e -> Log.warn (fun m -> m "Error getting job: %a" pp_error e))
     >>= fun (readme, builds) ->
+    if is_accept_json req then
+      Views.Job.make_json ~failed:true ~job_name ~platform ~readme builds
+      |> Yojson.Basic.to_string
+      |> Dream.json |> Lwt_result.ok
+    else
       Views.Job.make ~failed:true ~job_name ~platform ~readme builds
       |> string_of_html |> Dream.html |> Lwt_result.ok
   in

lib/views.ml

@@ -374,6 +374,23 @@ have questions or suggestions.
        @ make_failed_builds
        @ make_all_or_active all)
 
+  let make_json ~all:_ section_job_map =
+    let all_jobs =
+      Utils.String_map.fold
+        (fun _section jobs acc ->
+           List.map (fun (job_name, _, _) -> `String job_name) jobs @ acc)
+        section_job_map []
+    in
+    let by_section =
+      Utils.String_map.fold
+        (fun section jobs acc ->
+           (section, `List (List.map (fun (job_name, _, _) -> `String job_name) jobs)) :: acc)
+        section_job_map []
+    in
+    `Assoc [
+      "jobs", `List all_jobs;
+      "jobs_by_section", `Assoc by_section;
+    ]
 end
 
 module Job = struct
@@ -452,7 +469,28 @@ module Job = struct
     let title = Fmt.str "Job %s %a" job_name pp_platform platform in
     layout ~nav ~title @@ make_body ~failed ~job_name ~platform ~readme builds
 
-
+  let make_json ~failed:_ ~job_name:_ ~platform:_ ~readme:_ builds =
+    (* For now we will ignore most arguments. It's to keep the arguments the
+       same as [make]. This is subject to change. *)
+    let build (build, main_binary) =
+      let main_binary =
+        match main_binary with
+        | None -> `Null
+        | Some { Builder_db.filepath; sha256; size } ->
+          `Assoc [
+            "filename", `String (Fpath.basename filepath);
+            "sha256", `String (Ohex.encode sha256);
+            "size", `Int size;
+          ]
+      in
+      `Assoc [
+        "uuid", `String (Uuidm.to_string build.Builder_db.Build.uuid);
+        "main_binary", main_binary
+      ]
+    in
+    `Assoc [
+      "builds", `List (List.map build builds);
+    ]
 end
 
 module Job_build = struct

populate-local.sh

@@ -0,0 +1,30 @@
+#!/bin/sh
+
+# Edit these values to your needs:
+remote_instance=https://builds.robur.coop
+local_user_pass=test:test
+local_instance="http://${local_user_pass}@localhost:3000"
+limit=100
+
+curl_json () {
+	curl --silent --fail --location --header "Accept: application/json" "$@"
+}
+
+curl_json "${remote_instance}/" | jq -r .jobs[] | {
+	while read -r job_name; do
+		curl_json "${remote_instance}/job/${job_name}" | jq -r .builds[].uuid | {
+			while read -r build_uuid; do
+				if [ "$limit" -eq 0 ]; then
+					break 2;
+				fi
+				dest=$(mktemp "builder-${build_uuid}.XXXXXXXXXX")
+				curl --silent --fail "${remote_instance}/job/${job_name}/build/${build_uuid}/exec" > "$dest" && {
+					echo "Uploading $job_name build $build_uuid"
+                                        curl --data-binary "@${dest}" "${local_instance}/upload"
+				}
+				rm -f "$dest"
+				limit=$((limit - 1))
+			done
+		}
+	done
+}