Limit the number of concurrent downloads

At the moment this is hardcoded to 20 concurrent downloads.
This commit is contained in:
Reynir Björnsson 2022-08-30 16:25:02 +02:00
parent 236e09b7e0
commit 030e6d8ab7

View file

@ -369,7 +369,9 @@ module Make
Logs.info (fun m -> m "git: %s" msg);
Git.find_urls store >>= fun urls ->
Disk.init kv >>= fun disk ->
let pool = Lwt_pool.create 20 (Fun.const Lwt.return_unit) in
Lwt_list.iter_p (fun (url, csums) ->
Lwt_pool.use pool @@ fun () ->
HM.fold (fun h v r ->
r >>= function
| true -> Disk.exists disk h (hex_to_string v)