diff --git a/mirage/bad.ml b/mirage/bad.ml new file mode 100644 index 0000000..249b338 --- /dev/null +++ b/mirage/bad.ml @@ -0,0 +1,194 @@ +let archives = + let too_big = + [ "https://github.com/Opsian/opsian-ocaml/releases/download/0.1/0.1.tar.gz" ] + + and hash_mismatch = [ + "http://cdn.skylable.com/source/libres3-1.3.tar.gz" ; + "http://cdn.skylable.com/source/libres3-0.3.tar.gz" ; + "http://cdn.skylable.com/source/libres3-1.2.tar.gz" ; + "http://cdn.skylable.com/source/libres3-0.9.tar.gz" ; + "http://cdn.skylable.com/source/libres3-0.2.tar.gz" ; + "http://cdn.skylable.com/source/libres3-1.0.tar.gz" ; + "http://cdn.skylable.com/source/libres3-1.1.tar.gz" ; + "http://cdn.skylable.com/source/libres3-0.1.tar.gz" ; + "https://github.com/lemaetech/http-cookie/releases/download/v3.0.0/http-cookie-v3.0.0.tbz" ; + "http://oqamldebug.forge.ocamlcore.org/oqamldebug-0.9.4.tar.gz" ; + "http://oqamldebug.forge.ocamlcore.org/oqamldebug-0.9.2.tar.gz" ; + "http://oqamldebug.forge.ocamlcore.org/oqamldebug-0.9.3.tar.gz" ; + "http://oqamldebug.forge.ocamlcore.org/oqamldebug-0.9.5.tar.gz" ; + "http://oqamldebug.forge.ocamlcore.org/oqamldebug-0.9.1.tar.gz" ; + "https://github.com/OCamlPro/ezjs_fetch/archive/0.1.tar.gz" ; + "http://github.com/OCamlPro/typerex-build/archive/1.99.13-beta.tar.gz" ; + "https://github.com/mirage/dyntype/tarball/dyntype-0.8.5" ; + "https://github.com/mirage/dyntype/tarball/dyntype-0.8.3" ; + "https://github.com/mirage/dyntype/tarball/dyntype-0.8.2" ; + "https://github.com/mirage/dyntype/tarball/dyntype-0.8.4" ; + "https://github.com/mirage/mirage-http-unix/archive/v1.0.0.tar.gz" ; + "http://github.com/OCamlPro/typerex-build/archive/1.99.15-beta.tar.gz" ; + "http://github.com/OCamlPro/typerex-build/archive/1.99.14-beta.tar.gz" ; + "https://github.com/paulpatault/ocamlog/archive/v0.1.tar.gz" ; + "https://github.com/pveber/OCaml-R/archive/pre-nyc-refactoring.tar.gz" ; + "https://github.com/paulpatault/ocamlog/archive/v0.2.tar.gz" ; + "http://github.com/OCamlPro/typerex-build/archive/1.99.16-beta.tar.gz" ; + "https://github.com/FStarLang/kremlin/archive/v0.9.6.0.zip" ; + "https://gitlab.com/dailambda/plebeia/-/archive/2.0.2/plebeia-2.0.2.tar.gz" ; + "https://github.com/mirleft/ocaml-tls/archive/0.5.0.tar.gz" ; + "https://github.com/eth-sri/ELINA/archive/1.3.tar.gz" ; + "https://gitlab.com/trustworthy-refactoring/refactorer/-/archive/0.1/refactorer-0.1.zip" ; + "https://github.com/completium/archetype-lang/archive/1.3.3.tar.gz" ; + "https://github.com/chetmurthy/pa_ppx/archive/0.01.tar.gz" ; + "https://github.com/chambart/ocaml-1/archive/lto.tar.gz" ; + "https://github.com/Kappa-Dev/KaSim/archive/v3.5-250915.tar.gz" ; + "https://github.com/bsansouci/bsb-native/archive/1.9.4.tar.gz" + ] + + and bad_request = [ + "http://cgit.freedesktop.org/cairo-ocaml/snapshot/cairo-ocaml-1.2.0.tar.gz" + ] + + and not_found = [ + "http://pw374.github.io/distrib/frag/frag-0.1.0.tar.gz" ; + "http://pw374.github.io/distrib/glical/glical-0.0.3.tar.gz" ; + "http://pw374.github.io/distrib/glical/glical-0.0.4.tar.gz" ; + "http://pw374.github.io/distrib/glical/glical-0.0.1.tar.gz" ; + "http://pw374.github.io/distrib/glical/glical-0.0.2.tar.gz" ; + "http://pw374.github.io/distrib/glical/glical-0.0.5.tar.gz" ; + "http://pw374.github.io/distrib/glical/glical-0.0.7.tar.gz" ; + "http://pw374.github.io/distrib/mpp/mpp-0.1.1.tar.gz" ; + "http://pw374.github.io/distrib/mpp/mpp-0.1.0.tar.gz" ; + "http://pw374.github.io/distrib/mpp/mpp-0.1.2.tar.gz" ; + "http://pw374.github.io/distrib/mpp/mpp-0.1.7.tar.gz" ; + "http://pw374.github.io/distrib/mpp/mpp-0.1.3.tar.gz" ; + "http://pw374.github.io/distrib/mpp/mpp-0.1.8.tar.gz" ; + "http://pw374.github.io/distrib/mpp/mpp-0.1.4.tar.gz" ; + "http://pw374.github.io/distrib/mpp/mpp-0.1.5.tar.gz" ; + "http://pw374.github.io/distrib/mpp/mpp-0.2.0.tar.gz" ; + "http://pw374.github.io/distrib/mpp/mpp-0.2.1.tar.gz" ; + "http://pw374.github.io/distrib/mpp/mpp-0.3.0.tar.gz" ; + "http://pw374.github.io/distrib/omd/omd-0.3.tar.gz" ; + "http://pw374.github.io/distrib/omd/omd-0.4.tar.gz" ; + "http://pw374.github.io/distrib/omd/omd-0.5.4.tar.gz" ; + "http://pw374.github.io/distrib/omd/omd-0.5.5.tar.gz" ; + "http://pw374.github.io/distrib/omd/omd-0.5.tar.gz" ; + "http://pw374.github.io/distrib/omd/omd-0.6.0.tar.gz" ; + "http://pw374.github.io/distrib/omd/omd-0.6.2.tar.gz" ; + "http://pw374.github.io/distrib/omd/omd-0.6.3.tar.gz" ; + "http://pw374.github.io/distrib/omd/omd-0.6.4.tar.gz" ; + "http://pw374.github.io/distrib/omd/omd-0.6.5.tar.gz" ; + "http://pw374.github.io/distrib/omd/omd-0.7.0.tar.gz" ; + "http://pw374.github.io/distrib/omd/omd-0.7.1.tar.gz" ; + "http://pw374.github.io/distrib/omd/omd-0.7.2.tar.gz" ; + "http://pw374.github.io/distrib/omd/omd-0.7.4.tar.gz" ; + "http://pw374.github.io/distrib/omd/omd-0.7.3.tar.gz" ; + "http://pw374.github.io/distrib/omd/omd-0.7.5.tar.gz" ; + "http://pw374.github.io/distrib/omd/omd-0.8.2.tar.gz" ; + "http://pw374.github.io/distrib/omd/omd-0.8.0.tar.gz" ; + "http://pw374.github.io/distrib/omd/omd-0.8.1.tar.gz" ; + "http://pw374.github.io/distrib/omd/omd-0.9.0.tar.gz" ; + "http://pw374.github.io/distrib/omd/omd-0.9.1.tar.gz" ; + "http://pw374.github.io/distrib/omd/omd-0.9.7.tar.gz" ; + "http://pw374.github.io/distrib/omd/omd-1.0.0.tar.gz" ; + "http://pw374.github.io/distrib/omd/omd-1.0.1.tar.gz" ; + "http://pw374.github.io/distrib/omd/omd-1.1.0.tar.gz" ; + "http://pw374.github.io/distrib/omd/omd-1.1.1.tar.gz" ; + "http://pw374.github.io/distrib/omd/omd-1.1.2.tar.gz" ; + "http://pw374.github.io/distrib/omd/omd-1.2.0.tar.gz" ; + "http://pw374.github.io/distrib/omd/omd-1.2.1.tar.gz" ; + "http://pw374.github.io/distrib/omd/omd-1.2.2.tar.gz" ; + "http://pw374.github.io/distrib/omd/omd-1.2.4.tar.gz" ; + "http://pw374.github.io/distrib/omd/omd-1.2.5.tar.gz" ; + "http://pw374.github.io/distrib/omd/omd-1.2.6.tar.gz" ; + "http://pw374.github.io/distrib/omd/omd-1.2.3.tar.gz" ; + "http://pw374.github.io/distrib/omd/omd-1.3.0.tar.gz" ; + "http://zoggy.github.com/ocamldot/ocamldot-1.0.tar.gz" ; + "http://zoggy.github.io/stog/stog-0.4.tar.gz" ; + "http://zoggy.github.io/genet/genet-0.6.tar.gz" ; + "http://pw374.github.io/distrib/omd/omd-0.6.1.tar.gz" ; + "http://pw374.github.io/distrib/omd/omd-0.9.4.tar.gz" ; + "http://pw374.github.io/distrib/omd/omd-0.9.6.tar.gz" ; + "http://pw374.github.io/distrib/omd/omd-0.9.5.tar.gz" ; + "http://pw374.github.io/distrib/omd/omd-0.9.3.tar.gz" ; + "http://pw374.github.io/distrib/omd/omd-1.1.3.tar.gz" ; + "http://coccinelle.lip6.fr/distrib/coccinelle-1.0.0-rc22.tgz" ; + "http://coccinelle.lip6.fr/distrib/coccinelle-1.0.0-rc21.tgz" ; + "http://coccinelle.lip6.fr/distrib/coccinelle-1.0.0.tgz" ; + "http://proverif.inria.fr/proverif1.96pl1.tar.gz" ; + "http://proverif.inria.fr/proverif1.97.tar.gz" ; + "http://proverif.inria.fr/proverif1.98.tar.gz" ; + "http://proverif.inria.fr/proverif1.97pl3.tar.gz" ; + "http://proverif.inria.fr/proverif1.98pl1.tar.gz" ; + "http://proverif.inria.fr/proverif1.97pl1.tar.gz" ; + "https://github.com/jrochel/eliom/archive/6.4.0.tar.gz" ; + "https://github.com/drjdn/ocaml_lua_parser/archive/1.0.1.tar.gz" ; + "https://github.com/sagotch/To.ml/archive/v1.0.0.tar.gz" ; + "https://github.com/roburio/ocaml-solo5-elftool/releases/download/v0.1.0/solo5-elftool-0.1.0.tbz" ; + "https://zoggy.github.io/ocaml-rdf/ocaml-rdf-0.9.0.tar.gz" ; + "https://github.com/sagotch/To.ml/archive/v2.1.0.tar.gz" ; + "https://github.com/sagotch/To.ml/archive/v2.0.0.tar.gz" ; + "https://zoggy.github.io/ocaml-taglog/taglog-0.1.0.tar.gz" ; + "https://zoggy.github.io/ocaml-taglog/taglog-0.2.0.tar.gz" ; + "https://zoggy.github.io/ocf/ocf-0.3.0.tar.gz" ; + "https://zoggy.github.io/ojs-base/ojs-base-0.1.0.tar.gz" ; + "https://zoggy.github.io/stog/plugins/stog-writing-0.8.0.tar.gz" ; + "https://zoggy.github.io/stog/stog-0.13.0.tar.gz" ; + "https://zoggy.github.io/ocaml-taglog/taglog-0.3.0.tar.gz" ; + "https://zoggy.github.io/ocf/ocf-0.1.0.tar.gz" ; + "https://opam.ocaml.org/cache/md5/24/24b163eb77e6832747dccd6cc8a5d57c" ; + ] + + and forbidden = [ + "https://gforge.inria.fr/frs/download.php/33440/heptagon-1.00.06.tar.gz" ; + "https://gforge.inria.fr/frs/download.php/file/33677/dose3-3.2.2.tar.gz" ; + "https://gforge.inria.fr/frs/download.php/file/34920/javalib-2.3.1.tar.bz2" ; + "https://gforge.inria.fr/frs/download.php/file/36092/javalib-2.3.2.tar.bz2" ; + "https://gforge.inria.fr/frs/download.php/file/36093/sawja-1.5.2.tar.bz2" ; + "https://gforge.inria.fr/frs/download.php/file/37154/javalib-2.3.4.tar.bz2" ; + "https://gforge.inria.fr/frs/download.php/file/37403/sawja-1.5.3.tar.bz2" ; + "https://gforge.inria.fr/frs/download.php/file/36307/javalib-2.3.3.tar.bz2" ; + "https://gforge.inria.fr/frs/download.php/file/37655/javalib-2.3.5.tar.bz2" ; + "https://gforge.inria.fr/frs/download.php/file/37656/sawja-1.5.4.tar.bz2" ; + "https://gforge.inria.fr/frs/download.php/file/34921/sawja-1.5.1.tar.bz2" ; + ] + + and three_o_o = [ + "https://github.com/Gbury/dolmen/archive/v0.4.tar.gz" ; + "https://github.com/Stevendeo/Pilat/archive/1.3.tar.gz" ; + "https://github.com/OCamlPro/ocp-indent/archive/1.5.tar.gz" ; + "https://github.com/backtracking/combine/archive/release-0.6.zip" ; + "https://github.com/cakeplus/pa_comprehension/archive/0.4.tar.gz" ; + "https://github.com/cakeplus/mparser/archive/1.0.tar.gz" ; + "https://github.com/chenyukang/rubytt/archive/v0.1.tar.gz" ; + "https://github.com/cakeplus/pa_where/archive/0.4.tar.gz" ; + "https://github.com/metaocaml/ber-metaocaml/archive/ber-n102.tar.gz" ; + "https://github.com/cakeplus/pa_solution/archive/0.5.tar.gz" ; + "https://github.com/cakeplus/mparser/archive/1.2.1.tar.gz" ; + "https://github.com/cakeplus/pa_solution/archive/0.7.tar.gz" ; + "https://github.com/cakeplus/pa_solution/archive/0.6.tar.gz" ; + "https://github.com/mirage/mirage-tcpip/archive/v2.8.1.tar.gz" ; + "https://github.com/modlfo/pla/archive/v1.4.tar.gz" ; + "https://github.com/murmour/pa_qualified/archive/0.5.tar.gz" ; + "https://github.com/ocaml-ppx/ocamlformat/archive/v0.2.tar.gz" ; + "https://github.com/murmour/pa_qualified/archive/0.6.tar.gz" ; + "https://github.com/ocaml-ppx/ocamlformat/archive/support.0.2.tar.gz" ; + "https://github.com/ocaml/oloop/archive/0.1.2.tar.gz" ; + "https://github.com/cakeplus/mparser/archive/1.0.1.tar.gz" ; + "https://github.com/cakeplus/mparser/archive/1.1.tar.gz" ; + "https://github.com/savonet/ocaml-ffmpeg/archive/v1.0.0-rc1.tar.gz" ; + "https://github.com/ocaml/opam2web/archive/2.0.tar.gz" ; + "https://github.com/savonet/ocaml-ffmpeg/archive/v1.0.0.tar.gz" ; + ] + + and five_o_three = [ "https://gitlab.com/gasche/build_path_prefix_map/repository/0.2/archive.tar.gz" ] + + and is_ftp = [ "ftp://ftp.netbsd.org/pub/pkgsrc/distfiles/wyrd-1.4.6.tar.gz" ] + + and connect_fails = [ + "http://godi.0ok.org/godi-backup/shcaml-0.1.3.tar.gz" ; + "http://www.first.in-berlin.de/software/tools/apalogretrieve/apalogretrieve-0-9-6_4.tgz" ; + "https://cavale.enseeiht.fr/osdp/osdp-0.5.4.tgz" ; + "https://cavale.enseeiht.fr/osdp/osdp-0.6.0.tgz" ; + "https://cavale.enseeiht.fr/osdp/osdp-1.0.0.tgz" ; + ] + in + + too_big @ hash_mismatch @ bad_request @ not_found @ forbidden @ three_o_o @ five_o_three @ is_ftp @ connect_fails diff --git a/mirage/config.ml b/mirage/config.ml index 5eda798..71885b2 100644 --- a/mirage/config.ml +++ b/mirage/config.ml @@ -55,9 +55,16 @@ let sectors_cache = let doc = Key.Arg.info ~doc ["sectors-cache"] in Key.(create "sectors-cache" Arg.(opt int64 Int64.(mul 4L 2048L) doc)) +let sectors_git = + let doc = "Number of sectors reserved for git dump." in + let doc = Key.Arg.info ~doc ["sectors-git"] in + Key.(create "sectors-git" Arg.(opt int64 Int64.(mul 40L (mul 2L 1024L)) doc)) + let mirror = foreign "Unikernel.Make" - ~keys:[ Key.v check ; Key.v verify ; Key.v remote ; Key.v parallel_downloads ; Key.v hook_url ; Key.v tls_authenticator ; Key.v port ; Key.v sectors_cache ] + ~keys:[ Key.v check ; Key.v verify ; Key.v remote ; + Key.v parallel_downloads ; Key.v hook_url ; Key.v tls_authenticator ; + Key.v port ; Key.v sectors_cache ; Key.v sectors_git ; ] ~packages:[ package ~min:"0.1.0" ~sublibs:[ "mirage" ] "paf" ; package "h2" ; diff --git a/mirage/unikernel.ml b/mirage/unikernel.ml index f2e825a..496d038 100644 --- a/mirage/unikernel.ml +++ b/mirage/unikernel.ml @@ -276,7 +276,6 @@ module Make (* on disk, we use a flat file system where the filename is the sha256 of the data *) let init ~verify dev dev_md5s dev_sha512s = - Logs.info (fun m -> m "init with verify %B" verify); KV.list dev Mirage_kv.Key.empty >>= function | Error e -> Logs.err (fun m -> m "error %a listing kv" KV.pp_error e); assert false | Ok entries -> @@ -716,13 +715,11 @@ stamp: %S end + let bad_archives = SSet.of_list Bad.archives + let download_archives disk http_ctx store = Git.find_urls store >>= fun urls -> - let urls = SM.filter (fun k _ -> - not (String.equal - "https://github.com/Opsian/opsian-ocaml/releases/download/0.1/0.1.tar.gz" - k)) urls - in + let urls = SM.filter (fun k _ -> not (SSet.mem k bad_archives)) urls in let pool = Lwt_pool.create (Key_gen.parallel_downloads ()) (Fun.const Lwt.return_unit) in let idx = ref 0 in Lwt_list.iter_p (fun (url, csums) -> @@ -757,22 +754,57 @@ stamp: %S Disk.update_caches disk >|= fun () -> Logs.info (fun m -> m "downloading of %d urls done" (SM.cardinal urls)) + let dump_git git_dump git_kv = + Git_kv.to_octets git_kv >>= fun data -> + Cache.write git_dump data >|= function + | Ok () -> + Logs.info (fun m -> m "dumped git %d bytes" (String.length data)) + | Error e -> + Logs.warn (fun m -> m "failed to dump git: %a" Cache.pp_write_error e) + + let restore_git git_dump git_ctx = + Cache.read git_dump >>= function + | Ok None -> Lwt.return (Error ()) + | Error e -> + Logs.warn (fun m -> m "failed to read git state: %a" Cache.pp_error e); + Lwt.return (Error ()) + | Ok Some data -> + Git_kv.of_octets git_ctx ~remote:(Key_gen.remote ()) data >|= function + | Ok git_kv -> Ok git_kv + | Error `Msg msg -> + Logs.err (fun m -> m "error restoring git state: %s" msg); + Error () + module Paf = Paf_mirage.Make(Time)(Stack.TCP) let start block _time _pclock stack git_ctx http_ctx = BLOCK.get_info block >>= fun info -> let sectors_cache = Key_gen.sectors_cache () in - Part.connect Int64.(sub info.size_sectors (mul 2L sectors_cache)) block >>= fun (b1, rest) -> + let sectors_git = Key_gen.sectors_git () in + let git_start = + let cache_size = Int64.(mul 2L sectors_cache) in + Int64.(sub info.size_sectors (add cache_size sectors_git)) + in + Part.connect git_start block >>= fun (b1, rest) -> + let git_dump, rest = Part.subpartition sectors_git rest in let b2, b3 = Part.subpartition sectors_cache rest in KV.connect b1 >>= fun kv -> Cache.connect b2 >>= fun md5s -> Cache.connect b3 >>= fun sha512s -> + Cache.connect git_dump >>= fun git_dump -> Logs.info (fun m -> m "Available bytes in tar storage: %Ld" (KV.free kv)); Disk.init ~verify:(Key_gen.verify ()) kv md5s sha512s >>= fun disk -> if Key_gen.check () then Lwt.return_unit else - Git_kv.connect git_ctx (Key_gen.remote ()) >>= fun git_kv -> + begin + restore_git git_dump git_ctx >>= function + | Ok git_kv -> Lwt.return git_kv + | Error () -> + Git_kv.connect git_ctx (Key_gen.remote ()) >>= fun git_kv -> + dump_git git_dump git_kv >|= fun () -> + git_kv + end >>= fun git_kv -> Serve.commit_id git_kv >>= fun commit_id -> Logs.info (fun m -> m "git: %s" commit_id); Serve.create git_kv >>= fun serve -> @@ -780,7 +812,9 @@ stamp: %S let update () = Serve.update_git serve git_kv >>= function | None | Some [] -> Lwt.return_unit - | Some _changes -> download_archives disk http_ctx git_kv + | Some _changes -> + dump_git git_dump git_kv >>= fun () -> + download_archives disk http_ctx git_kv in let service = Paf.http_service