robur/opam-mirror
4
0
Fork 0
Code Issues 6 Pull requests 2 Projects Releases Packages Wiki Activity Actions

Merge pull request 'if --verify is passed, don't use the md5s/sha512s from disk, instead re-create them' (#24) from trash-md5-sha512-on-verify into main

Browse source 
Reviewed-on: https://git.robur.io/robur/opam-mirror/pulls/24
This commit is contained in:
Hannes Mehnert 2022-10-28 12:59:33 +00:00
commit 878ecab0b2
2 changed files with 15 additions and 33 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

10
mirage/config.ml
View file

@ -9,11 +9,13 @@ let check =
in
Key.(create "check" Arg.(flag doc))
let verify =
let verify_sha256 =
let doc =
Key.Arg.info ~doc:"Verify the cache contents" ["verify"]
Key.Arg.info ~doc:"Verify the SHA256 checksums of the cache contents, and \
re-build the other checksum caches."
["verify-sha256"]
in
Key.(create "verify" Arg.(flag doc))
Key.(create "verify-sha256" Arg.(flag doc))
let remote =
let doc =
@ -67,7 +69,7 @@ let ignore_local_git =
let mirror =
foreign "Unikernel.Make"
~keys:[ Key.v check ; Key.v verify ; Key.v remote ;
~keys:[ Key.v check ; Key.v verify_sha256 ; Key.v remote ;
Key.v parallel_downloads ; Key.v hook_url ; Key.v tls_authenticator ;
Key.v port ; Key.v sectors_cache ; Key.v sectors_git ;
Key.v ignore_local_git ;

38
mirage/unikernel.ml
View file

@ -276,19 +276,23 @@ module Make
read_more a 0
(* on disk, we use a flat file system where the filename is the sha256 of the data *)
let init ~verify dev dev_md5s dev_sha512s =
let init ~verify_sha256 dev dev_md5s dev_sha512s =
KV.list dev Mirage_kv.Key.empty >>= function
| Error e -> Logs.err (fun m -> m "error %a listing kv" KV.pp_error e); assert false
| Ok entries ->
let t = empty dev dev_md5s dev_sha512s in
Cache.read t.dev_md5s >>= fun r ->
(match r with
| Ok Some s -> Result.iter (fun md5s -> t.md5s <- md5s) (unmarshal_sm s)
| Ok Some s ->
if not verify_sha256 then
Result.iter (fun md5s -> t.md5s <- md5s) (unmarshal_sm s)
| Ok None -> Logs.debug (fun m -> m "No md5s cached")
| Error e -> Logs.warn (fun m -> m "Error reading md5s cache: %a" Cache.pp_error e));
Cache.read t.dev_sha512s >>= fun r ->
(match r with
| Ok Some s -> Result.iter (fun sha512s -> t.sha512s <- sha512s) (unmarshal_sm s)
| Ok Some s ->
if not verify_sha256 then
Result.iter (fun sha512s -> t.sha512s <- sha512s) (unmarshal_sm s)
| Ok None -> Logs.debug (fun m -> m "No sha512s cached")
| Error e -> Logs.warn (fun m -> m "Error reading sha512s cache: %a" Cache.pp_error e));
let md5s = SSet.of_list (List.map snd (SM.bindings t.md5s))
@ -303,7 +307,7 @@ module Make
| `Value ->
let open Mirage_crypto.Hash in
let sha256_final =
if verify then
if verify_sha256 then
let f s =
let digest = SHA256.get s in
if not (String.equal name (to_hex digest)) then
@ -321,18 +325,6 @@ module Make
t.md5s <- SM.add (to_hex digest) name t.md5s
in
Some f
else if verify then
let f s =
let digest = MD5.get s |> to_hex in
match SM.find_opt digest t.md5s with
| Some x when String.equal name x -> ()
| y ->
Logs.err (fun m -> m "corrupt MD5 data for %s, \
expected %a, computed %s"
name Fmt.(option ~none:(any "NONE") string) y
digest)
in
Some f
else
None
and sha512_final =
@ -342,18 +334,6 @@ module Make
t.sha512s <- SM.add (to_hex digest) name t.sha512s
in
Some f
else if verify then
let f s =
let digest = SHA512.get s |> to_hex in
match SM.find_opt digest t.sha512s with
| Some x when String.equal name x -> ()
| y ->
Logs.err (fun m -> m "corrupt SHA512 data for %s, \
expected %a, computed %s"
name Fmt.(option ~none:(any "NONE") string) y
digest)
in
Some f
else
None
in
@ -791,7 +771,7 @@ stamp: %S
Cache.connect sha512s >>= fun sha512s ->
Cache.connect git_dump >>= fun git_dump ->
Logs.info (fun m -> m "Available bytes in tar storage: %Ld" (KV.free kv));
Disk.init ~verify:(Key_gen.verify ()) kv md5s sha512s >>= fun disk ->
Disk.init ~verify_sha256:(Key_gen.verify_sha256 ()) kv md5s sha512s >>= fun disk ->
if Key_gen.check () then
Lwt.return_unit
else