diff --git a/src/dev_cache.erl b/src/dev_cache.erl index 7f49736b6..5dde59030 100644 --- a/src/dev_cache.erl +++ b/src/dev_cache.erl @@ -3,10 +3,87 @@ %%% supports writing messages to the store, if the node message has the %%% writer's address in its `cache_writers' key. -module(dev_cache). --export([read/3, write/3, link/3, read_from_cache/2]). +-export([read/3, write/3, link/3, read_from_cache/2, expected_response/3]). -include("include/hb.hrl"). -include_lib("eunit/include/eunit.hrl"). +%% @doc An `is-admissible`-compliant key that verifies a `~cache@1.0/read` +%% response contains the message we expect, and that it is valid. Additionally, +%% if the `http-reference` key is set, we execute the `on/cache-valid-response` +%% hook. +expected_response(Base, Req, Opts) -> + maybe + {ok, Response} ?= hb_maps:find(<<"body">>, Req, Opts), + {ok, Expected} ?= hb_maps:find(<<"expected">>, Base, Opts), + true ?= check_response_matches_expected(Response, Expected, Opts), + dev_hook:on( + [<<"~cache@1.0">>, <<"admissible-response">>], + Response, + Opts + ), + {ok, true} + else Reason -> + ?event(debug_admissible, {expected_response_error, Reason}), + {ok, false} + end. + +%% @doc Verify that a response from a remote cache matches the expected ID. +%% There are three cases: +%% 1. The response is a raw binary (e.g., a direct content-addressed read +%% of a data blob written with `hb_cache:write/2'): the binary's SHA-256 +%% hash must match `Expected'. +%% 2. The response is a structured message with commitments, and `Expected' +%% is a commitment ID (the common case for data items): `Expected' must +%% be among the commitment IDs and the corresponding commitment must +%% verify. +%% 3. The response is a structured message but `Expected' is a path rather +%% than a commitment ID (e.g. `~scheduler@1.0/assignments//'): +%% we verify all committer-attributed commitments on the response. +check_response_matches_expected(Response, Expected, _Opts) when is_binary(Response) -> + % Accept either a bare ID or a `data/' path (the convention used by + % `hb_cache:write/2' for content-addressed binary blobs). In both cases, + % the blob's SHA-256 hash must match the expected ID. + BinID = + case binary:split(Expected, <<"/">>) of + [<<"data">>, ID] -> ID; + _ -> Expected + end, + case ?IS_ID(BinID) + andalso hb_util:human_id(hb_crypto:sha256(Response)) =:= BinID of + true -> true; + false -> binary_hash_mismatch + end; +check_response_matches_expected(Response, Expected, Opts) -> + maybe + {ok, Commitments} ?= hb_maps:find(<<"commitments">>, Response, Opts), + CommIDs = maps:keys(Commitments), + ?event(debug_admissible, + {expected_response, + {response, Response}, + {expected, Expected}, + {commitments, CommIDs} + } + ), + true ?= + (not ?IS_ID(Expected) + orelse lists:member(Expected, CommIDs)) + orelse expected_id_not_found, + {ok, OnlyCommitted} = hb_message:with_only_committed(Response, Opts), + % For ID-based reads, verify the specific commitment that claims to be + % the requested ID. For non-ID reads (path-based reads), the `Expected' + % value is not itself a commitment ID, so we instead verify all + % committer-attributed commitments on the response. + VerifyReq = + case ?IS_ID(Expected) of + true -> #{ <<"commitment-ids">> => [Expected] }; + false -> #{ <<"committers">> => <<"all">> } + end, + true ?= + hb_message:verify(OnlyCommitted, VerifyReq, Opts) + orelse invalid_commitment, + true + end. + %% @doc Read data from the cache. %% Retrieves data corresponding to a key from a local store. %% The key is extracted from the incoming message under <<"target">>. diff --git a/src/dev_codec_httpsig.erl b/src/dev_codec_httpsig.erl index 8e9635a13..4f98e72bf 100644 --- a/src/dev_codec_httpsig.erl +++ b/src/dev_codec_httpsig.erl @@ -575,6 +575,32 @@ validate_large_message_from_http_test() -> ?assert(hb_message:verify(OnlyCommitted, all, Opts)), ?event({msg_with_only_committed_verifies_hmac, <<"hmac-sha256">>}). +%% @doc Ensure that a signed response that contains both `status' and another +%% top-level locally-typed key (e.g. an integer) round-trips through HTTP and +%% verifies on the receiving side. This exercises the case where `ao-types' +%% must agree between signer and verifier for multiple locally-typed fields. +validate_sibling_typed_key_over_http_test() -> + Node = hb_http_server:start_node(Opts = #{ + force_signed => true, + commitment_device => <<"httpsig@1.0">>, + % Top-level locally-typed siblings to `status' in the signed response: + % one integer that sorts before `status' alphabetically, one that sorts + % after, and an atom. All become entries in `ao-types' alongside + % `status'. + <<"alpha-count">> => 7, + <<"test-count">> => 42, + <<"zebra-flag">> => true + }), + {ok, Res} = hb_http:get(Node, <<"/~meta@1.0/info">>, Opts), + ?event({received_with_sibling_typed, Res}), + ?assertEqual(7, hb_ao:get(<<"alpha-count">>, Res, undefined, Opts)), + ?assertEqual(42, hb_ao:get(<<"test-count">>, Res, undefined, Opts)), + ?assertEqual(true, hb_ao:get(<<"zebra-flag">>, Res, undefined, Opts)), + Signers = hb_message:signers(Res, Opts), + ?assert(length(Signers) == 1), + ?assert(hb_message:verify(Res, Signers, Opts)), + ?assert(hb_message:verify(Res, all, Opts)). + committed_id_test() -> Msg = #{ <<"basic">> => <<"value">> }, Opts = #{ priv_wallet => hb:wallet() }, diff --git a/src/dev_genesis_wasm.erl b/src/dev_genesis_wasm.erl index 9e1a8e1cd..a520eb5fc 100644 --- a/src/dev_genesis_wasm.erl +++ b/src/dev_genesis_wasm.erl @@ -191,10 +191,22 @@ ensure_started(Opts) -> IsRunning = is_genesis_wasm_server_running(Opts), IsCompiled = hb_features:genesis_wasm(), GenWASMProc = is_pid(hb_name:lookup(<<"genesis-wasm@1.0">>)), + ServerDirExists = filelib:is_dir(GenesisWasmServerDir), case IsRunning orelse (IsCompiled andalso GenWASMProc) of true -> % If it is, do nothing. true; + false when not ServerDirExists -> + % The sidecar isn't present on disk (e.g. a development host that + % hasn't fetched the `genesis-wasm-server' Node subproject). Report + % as unavailable rather than spawning an open_port that will die + % with `enoent' and leaving the caller hanging in the start poll. + ?event( + warning, + {ensure_started, genesis_wasm_server_missing, + GenesisWasmServerDir} + ), + false; false -> % The device is not running, so we need to start it. PID = diff --git a/src/dev_hook.erl b/src/dev_hook.erl index 1696745ec..1307b2e0a 100644 --- a/src/dev_hook.erl +++ b/src/dev_hook.erl @@ -92,7 +92,7 @@ find(HookName, Opts) -> find(#{}, #{ <<"target">> => <<"body">>, <<"body">> => HookName }, Opts). find(_Base, Req, Opts) -> HookName = maps:get(maps:get(<<"target">>, Req, <<"body">>), Req), - case maps:get(HookName, hb_opts:get(on, #{}, Opts), []) of + case hb_util:deep_get(HookName, hb_opts:get(on, #{}, Opts), [], Opts) of Handler when is_map(Handler) -> case hb_util:is_ordered_list(Handler, Opts) of true -> @@ -164,10 +164,8 @@ execute_handler(HookName, Handler, Req, Opts) -> % committed before execution. BaseReq = Req#{ - <<"path">> => - hb_maps:get(<<"path">>, Handler, HookName, Opts), - <<"method">> => - hb_maps:get(<<"method">>, Handler, <<"GET">>, Opts) + <<"path">> => hb_maps:get(<<"path">>, Handler, HookName, Opts), + <<"method">> => hb_maps:get(<<"method">>, Handler, <<"GET">>, Opts) }, CommitReqBin = hb_util:bin( diff --git a/src/dev_message.erl b/src/dev_message.erl index 857845e8e..24886c26e 100644 --- a/src/dev_message.erl +++ b/src/dev_message.erl @@ -271,9 +271,10 @@ commit(Self, Req, Opts) -> CommitOpts ), % Encode to a TABM + {ok, OnlyCommitted} = hb_message:with_only_committed(Base, Opts), Loaded = ensure_commitments_loaded( - hb_message:convert(Base, tabm, CommitOpts), + hb_message:convert(OnlyCommitted, tabm, CommitOpts), Opts ), {ok, Committed} = @@ -297,10 +298,11 @@ commit(Self, Req, Opts) -> verify(Self, Req, Opts) -> % Get the target message of the verification request. {ok, RawBase} = hb_message:find_target(Self, Req, Opts), + {ok, OnlyCommitted} = hb_message:with_only_committed(RawBase, Opts), Base = hb_message:convert( ensure_commitments_loaded( - RawBase, + OnlyCommitted, Opts ), tabm, diff --git a/src/dev_process_test_vectors.erl b/src/dev_process_test_vectors.erl index 98f90dac7..d1f9afa10 100644 --- a/src/dev_process_test_vectors.erl +++ b/src/dev_process_test_vectors.erl @@ -97,13 +97,21 @@ test_process() -> test_process(#{}). test_process(Opts) -> Wallet = hb:wallet(), + % Strip the base commitment before merging in the execution stack. A + % re-commit over a message that still carries a prior commitment will + % replicate that commitment's `committed' key list verbatim (see the + % "stacked commitments" branch of `keys_to_commit/3' in + % `dev_codec_httpsig'), leaving the newly-added `execution-device' + % and `device-stack' keys out of the signed set — and therefore out + % of the `with_only_committed' view used in compute. The + % `wasm_process'/`aos_process' helpers already follow this pattern. hb_message:commit( hb_maps:merge( - base_process(Opts), + hb_message:uncommitted(base_process(Opts), Opts), #{ <<"execution-device">> => <<"stack@1.0">>, <<"device-stack">> => [<<"test-device@1.0">>, <<"test-device@1.0">>] - }, + }, Opts ), Opts#{ priv_wallet => Wallet } diff --git a/src/dev_query.erl b/src/dev_query.erl index 4dc43a09d..776acc19b 100644 --- a/src/dev_query.erl +++ b/src/dev_query.erl @@ -50,16 +50,9 @@ graphql(Req, Base, Opts) -> %% @doc Return whether a GraphQL esponse in a message has transaction results. %% This key is used in HB's gateway client multirequest configuration to %% determine if the response from the node should be considered admissible. -has_results(Base, Req, Opts) -> - JSON = - hb_ao:get_first( - [ - {{as, <<"message@1.0">>, Base}, <<"body">>}, - {{as, <<"message@1.0">>, Req}, <<"body">>} - ], - <<"{}">>, - Opts - ), +has_results(_Base, RawReq, Opts) -> + Req = hb_maps:get(<<"body">>, RawReq, #{}, Opts), + JSON = hb_maps:get(<<"body">>, Req, <<>>, Opts), Decoded = hb_json:decode(JSON), ?event(debug_multi, {has_results, {decoded_json, Decoded}}), case Decoded of diff --git a/src/dev_scheduler.erl b/src/dev_scheduler.erl index 3d165342a..c7d343fc9 100644 --- a/src/dev_scheduler.erl +++ b/src/dev_scheduler.erl @@ -1533,12 +1533,14 @@ redirect_from_graphql_test_() -> {timeout, 60, fun redirect_from_graphql/0}. redirect_from_graphql() -> start(), + TestStore = hb_test_utils:test_store(), Opts = - #{ store => - [ - #{ <<"store-module">> => hb_store_fs, <<"name">> => <<"cache-mainnet">> }, - #{ <<"store-module">> => hb_store_gateway, <<"store">> => [] } - ] + #{ + store => + [ + TestStore, + #{ <<"store-module">> => hb_store_gateway } + ] }, {ok, Msg} = hb_cache:read(<<"0syT13r0s0tgPmIed95bJnuSqaD29HQNN8D3ElLSrsc">>, Opts), ?assertMatch( @@ -1603,14 +1605,12 @@ http_init() -> http_init(#{}). http_init(Opts) -> start(), Wallet = ar_wallet:new(), + TestStore = hb_test_utils:test_store(), ExtendedOpts = Opts#{ priv_wallet => Wallet, store => [ - #{ - <<"store-module">> => hb_store_volatile, - <<"name">> => <<"cache-TEST/volatile">> - }, - #{ <<"store-module">> => hb_store_gateway, <<"store">> => [] } + TestStore, + #{ <<"store-module">> => hb_store_gateway } ] }, Node = hb_http_server:start_node(ExtendedOpts), @@ -1663,17 +1663,18 @@ http_get_schedule(N, PMsg, From, To, Format) -> http_get_schedule_redirect_test_() -> {timeout, 60, fun http_get_schedule_redirect/0}. http_get_schedule_redirect() -> + start(), + TestStore = hb_test_utils:test_store(), Opts = #{ store => [ - #{ <<"store-module">> => hb_store_fs, <<"name">> => <<"cache-mainnet">> }, - #{ <<"store-module">> => hb_store_gateway, <<"opts">> => #{} } + TestStore, + #{ <<"store-module">> => hb_store_gateway } ], - scheduler_follow_redirects => false + scheduler_follow_redirects => false }, {N, _Wallet} = http_init(Opts), - start(), ProcID = <<"0syT13r0s0tgPmIed95bJnuSqaD29HQNN8D3ElLSrsc">>, Res = hb_http:get(N, <<"/", ProcID/binary, "/schedule">>, Opts), ?assertMatch({ok, #{ <<"location">> := Location }} when is_binary(Location), Res). diff --git a/src/dev_snp.erl b/src/dev_snp.erl index 48ce38a2b..a74b17ee1 100644 --- a/src/dev_snp.erl +++ b/src/dev_snp.erl @@ -439,11 +439,23 @@ extract_measurement_args(Msg, NodeOpts) -> -spec verify_report_integrity(ReportJSON :: binary()) -> {ok, true} | {error, report_signature_invalid}. verify_report_integrity(ReportJSON) -> - {ok, ReportIsValid} = dev_snp_nif:verify_signature(ReportJSON), - ?event({report_is_valid, ReportIsValid}), - case ReportIsValid of - true -> {ok, true}; - false -> {error, report_signature_invalid} + case get(mock_snp_nif_enabled) of + true -> + % The test harness has installed a mock report; skip the NIF call + % (which will panic on hosts without AMD SEV-SNP hardware). + {ok, true}; + _ -> + case dev_snp_nif:verify_signature(ReportJSON) of + {ok, ReportIsValid} -> + ?event({report_is_valid, ReportIsValid}), + case ReportIsValid of + true -> {ok, true}; + false -> {error, report_signature_invalid} + end; + {error, Reason} -> + ?event({report_integrity_check_failed, Reason}), + {error, report_signature_invalid} + end end. %% @doc Check if the node's debug policy is enabled. diff --git a/src/dev_test.erl b/src/dev_test.erl index 4a3632632..0dc2a7619 100644 --- a/src/dev_test.erl +++ b/src/dev_test.erl @@ -2,6 +2,7 @@ -export([info/3]). -export([info/1, test_func/1, compute/3, init/3, restore/3, snapshot/3, mul/2]). -export([mangle/3, update_state/3, increment_counter/3, delay/3]). +-export([log_request/3, logs/3]). -export([index/3, postprocess/3, load/3]). -include_lib("eunit/include/eunit.hrl"). -include("include/hb.hrl"). @@ -209,6 +210,57 @@ mangle(Base, _Req, Opts) -> end end. +%%% Logged messages functionality + +-define(LOG_PREFIX, "~test-device@1.0"). + +%% @doc Determines the store to use for logging requests. +determine_log_store(Base, _Req, Opts) -> + hb_maps:get( + <<"store">>, + Base, + hb_opts:get(store, no_viable_store, Opts), + Opts + ). + +%% @doc Write a pseudo-path to the store linking the received message to the +%% millisecond timestamp that the key was invoked. +log_request(Base, Req, Opts) -> + Timestamp = hb_util:bin(erlang:system_time(millisecond)), + Store = determine_log_store(Base, Req, Opts), + {ok, ReqID} = hb_cache:write(Req, Opts#{ store => Store }), + hb_store:make_link( + Store, + ReqID, + <> + ), + {ok, ReqID}. + +%% @doc Return all logs of requests to the device. +logs(Base, Req, Opts) -> + Store = determine_log_store(Base, Req, Opts), + LogOpts = Opts#{ store => Store }, + case hb_store:list(Store, <>) of + {ok, LogKeys} -> + Logs = + maps:from_list( + lists:map( + fun(K = <<"request-", TimeBin/binary>>) -> + {ok, Request} = + hb_cache:read( + <<"~test-device@1.0/", K/binary>>, + LogOpts + ), + {hb_util:int(TimeBin), Request} + end, + LogKeys + ) + ), + {ok, Logs}; + _ -> + {error, <<"No logs found.">>} + end. + %%% Tests %% @doc Tests the resolution of a default function. diff --git a/src/hb_http_multi.erl b/src/hb_http_multi.erl index c428c6b1d..5795acfa2 100644 --- a/src/hb_http_multi.erl +++ b/src/hb_http_multi.erl @@ -237,19 +237,26 @@ admissible_status(Status, Statuses) when is_list(Statuses) -> %% @doc If an `admissable` message is set for the request, check if the response %% adheres to it. Else, return `true'. admissible_response(_Response, undefined, _Opts) -> true; -admissible_response(Response, Msg, Opts) -> - Path = hb_maps:get(<<"path">>, Msg, <<"is-admissible">>, Opts), - Req = Response#{ <<"path">> => Path }, - Base = hb_message:without_unless_signed([<<"path">>], Msg, Opts), - ?event(debug_multi, - {executing_admissible_message, {message, Base}, {req, Req}} - ), - try hb_ao:resolve(Base, Req, Opts) of +admissible_response(Response, IsAdmissible, Opts) -> + Req = + IsAdmissible#{ + <<"path">> => + hb_maps:get( + <<"path">>, + IsAdmissible, + <<"is-admissible">>, + Opts + ), + <<"body">> => Response, + <<"http-reference">> => hb_opts:get(http_reference, undefined, Opts) + }, + ?event(debug_admissible, {admissible_response, {request, Req}, {opts, Opts}}), + try hb_ao:resolve(Req, Opts#{ hashpath => ignore }) of {ok, Res} when is_atom(Res) or is_binary(Res) -> - ?event(debug_multi, {admissible_result, {result, Res}}), + ?event(debug_admissible, {admissible_result, {result, Res}}), hb_util:atom(Res) == true; {error, Reason} -> - ?event(debug_multi, {admissible_error, {reason, Reason}}), + ?event(debug_admissible, {admissible_error, {reason, Reason}}), false catch Class:Reason:Stacktrace -> @@ -257,7 +264,7 @@ admissible_response(Response, Msg, Opts) -> {admissible_response, {class, Class}, {reason, Reason}, - {stacktrace, Stacktrace} + {stacktrace, {trace, Stacktrace}} } ), false diff --git a/src/hb_store_remote_node.erl b/src/hb_store_remote_node.erl index 90890e9c3..7845c0e54 100644 --- a/src/hb_store_remote_node.erl +++ b/src/hb_store_remote_node.erl @@ -28,6 +28,9 @@ scope(_StoreOpts) -> %% @returns The resolved key. resolve(#{ <<"node">> := Node }, Key) -> ?event({remote_resolve, {node, Node}, {key, Key}}), + Key; +resolve(#{ <<"nodes">> := Nodes }, Key) -> + ?event({remote_resolve, {nodes, Nodes}, {key, Key}}), Key. %% @doc Determine the type of value at a given key. @@ -37,8 +40,8 @@ resolve(#{ <<"node">> := Node }, Key) -> %% @param Opts A map of options (including node configuration). %% @param Key The key whose value type is determined. %% @returns simple if found, or not_found otherwise. -type(Opts = #{ <<"node">> := Node }, Key) -> - ?event({remote_type, {node, Node}, {key, Key}}), +type(Opts, Key) when is_map_key(<<"node">>, Opts); is_map_key(<<"nodes">>, Opts) -> + ?event({remote_type, {opts, Opts}, {key, Key}}), case read(Opts, Key) of not_found -> not_found; _ -> simple @@ -55,25 +58,75 @@ type(Opts = #{ <<"node">> := Node }, Key) -> read(#{ <<"only-ids">> := true }, Key) when not ?IS_ID(Key) -> not_found; read(Opts = #{ <<"node">> := Node }, Key) -> - ?event(store_remote_node, {executing_read, {node, Node}, {key, Key}}), + OptsWithoutNode = maps:remove(<<"node">>, Opts), + read(OptsWithoutNode#{ <<"nodes">> => [#{ <<"prefix">> => Node }] }, Key); +read(StoreOpts = #{ <<"nodes">> := Nodes }, Key) -> + MultirequestDirectives = + maps:filter( + fun(<<"multirequest-", _/binary>>, _) -> true; (_, _) -> false end, + StoreOpts + ), + ?event( + {read, + {nodes, Nodes}, + {key, Key}, + {multirequest_directives, MultirequestDirectives} + } + ), + HTTPReq = + maps:merge( + #{ + <<"method">> => <<"GET">>, + <<"path">> => <<"/~cache@1.0/read?target=", Key/binary>>, + <<"multirequest-responses">> => 1, + <<"multirequest-stop-after">> => true, + <<"multirequest-admissible">> => #{ + <<"device">> => <<"cache@1.0">>, + <<"path">> => <<"expected-response">>, + <<"expected">> => Key + } + }, + MultirequestDirectives + ), + % TODO: When `opts` key normalization lands, we should re-work this. + MaybeHooks = + case maps:find(<<"on">>, StoreOpts) of + {ok, Hooks} -> #{ on => Hooks }; + error -> #{} + end, + ?event({remote_read, {request, HTTPReq}, {hooks, MaybeHooks}}), HTTPRes = - hb_http:get( - Node, - #{ <<"path">> => <<"/~cache@1.0/read">>, <<"target">> => Key }, - Opts + hb_http:request( + HTTPReq, + MaybeHooks#{ + cache_control => [<<"no-cache">>, <<"no-store">>], + routes => + [ + #{ + <<"template">> => <<"/~cache@1.0/read">>, + <<"nodes">> => Nodes + } + ] + } ), - case HTTPRes of - {ok, Res} -> - % returning the whole response to get the test-key - {ok, Msg} = hb_message:with_only_committed(Res, Opts), - ?event(store_remote_node, {read_found, {result, Msg, response, Res}}), - maybe_cache(Opts, Msg, [Key]), - {ok, Msg}; - {error, _Err} -> - ?event(store_remote_node, {read_not_found, {key, Key}}), - not_found - end; -read(_, _) -> not_found. + handle_read_response(Key, HTTPRes, StoreOpts). + +%% @doc Handle a read response from the remote node, filtering the raw response +%% and invoking a possible local cache write operation. +handle_read_response(Key, {ok, Res}, StoreOpts) -> + {ok, Msg} = hb_message:with_only_committed(Res, StoreOpts), + ?event( + debug_admissible, + {remote_read, {only_committed, Msg}, {raw_response, Res}} + ), + maybe_cache(StoreOpts, Msg, [Key]), + {ok, Msg}; +handle_read_response(Key, UnexpectedRes, _StoreOpts) -> + ?event( + debug_admissible, + {read_failed, {key, Key}, {unexpected_response, UnexpectedRes}} + ), + not_found. %% @doc Cache the data if the cache is enabled. The `local-store' option may %% either be `false' or a store definition to use as the local cache. Additional @@ -203,6 +256,45 @@ make_group(_StoreOpts, _Path) -> not_found. %%% Tests %%%-------------------------------------------------------------------- +multinode_env() -> + Node1Store = [hb_test_utils:test_store()], + Node2Store = [hb_test_utils:test_store()], + Wallet1 = ar_wallet:new(), + Wallet2 = ar_wallet:new(), + Opts1 = #{ priv_wallet => Wallet1, store => Node1Store }, + Opts2 = #{ priv_wallet => Wallet2, store => Node2Store }, + Msg1 = hb_message:commit(#{ <<"key1">> => <<"message1">>, <<"num1">> => 1 }, Opts1), + Msg2 = hb_message:commit(#{ <<"key2">> => <<"message2">> }, Opts2), + BothMsg = + hb_message:commit( + #{ <<"key-both">> => <<"value-both">> }, + Opts1 + ), + {ok, ID1} = hb_cache:write(Msg1, Opts1), + {ok, ID2} = hb_cache:write(Msg2, Opts2), + {ok, IDBoth} = hb_cache:write(BothMsg, Opts1), + {ok, IDBoth} = hb_cache:write(BothMsg, Opts2), + Node1 = hb_http_server:start_node(Opts1), + Node2 = hb_http_server:start_node(Opts2), + RemoteStore = + #{ + <<"store-module">> => hb_store_remote_node, + <<"max-retries">> => 0, + <<"nodes">> => [ + #{ <<"prefix">> => Node1, <<"http-reference">> => <<"node1">> }, + #{ <<"prefix">> => Node2, <<"http-reference">> => <<"node2">> } + ], + <<"parallel">> => 1 + }, + #{ + ids_single => [ID1, ID2], + id_both => [IDBoth], + nodes => [Node1, Node2], + stores => [Node1Store, Node2Store], + opts => [Opts1, Opts2], + remote_store => RemoteStore + }. + %% @doc Test that we can create a store, write a random message to it, then %% start a remote node with that store, and read the message from it. read_test() -> @@ -240,15 +332,169 @@ read_only_ids_test() -> <<"message">>, #{ store => LocalStore } ), - Node = - hb_http_server:start_node( - #{ - store => LocalStore + Node = hb_http_server:start_node(#{ store => LocalStore }), + RemoteStore = + #{ + <<"store-module">> => hb_store_remote_node, + <<"node">> => Node, + <<"only-ids">> => true + }, + ?assertEqual(not_found, hb_cache:read(ID, #{ store => [RemoteStore] })). + +multiread_test() -> + #{ ids_single := [ID1, ID2], remote_store := RemoteStore } = multinode_env(), + ?assertMatch( + {ok, #{ <<"key1">> := <<"message1">>}}, + hb_cache:read(ID1, #{ store => RemoteStore }) + ), + ?assertMatch( + {ok, #{ <<"key2">> := <<"message2">>}}, + hb_cache:read(ID2, #{ store => RemoteStore }) + ). + +corrupted_id_test() -> + #{ + ids_single := [ID1|_], + stores := [Store1|_], + remote_store := RemoteStore + } = multinode_env(), + % Start by reading the message back and checking that it is accessible + % (and valid) to start with. + ?assertMatch( + {ok, #{ <<"key1">> := _ }}, + hb_cache:read(ID1, #{ store => RemoteStore }) + ), + {ok, Msg} = hb_cache:read(ID1, #{ store => Store1 }), + % Corrupt the value of `key1`, but keep the commitments. These commitments + % will now be invalid. A local store will return this invalid value, but + % a remote store will not. + hb_cache:write(Msg#{ <<"key1">> => <<"corrupt-value">> }, #{ store => Store1 }), + {ok, ReadCorruptMsg} = hb_cache:read(ID1, #{ store => Store1 }), + ?assertMatch( + #{ <<"key1">> := <<"corrupt-value">> }, + hb_cache:ensure_all_loaded(ReadCorruptMsg, #{ store => Store1 }) + ), + ?assertMatch(not_found, hb_cache:read(ID1, #{ store => RemoteStore })). + +multiread_corrupted_id_test() -> + #{ + id_both := IDBoth, + stores := [Store1, Store2], + remote_store := RemoteStore + } = multinode_env(), + % Force an invalid link on one node to the nessage stored in both nodes. + FakeID = hb_util:human_id(<<0:256>>), + ok = hb_store:make_link(Store1, IDBoth, FakeID), + % Check we can read the message back from the store locally. This would be + % a local node store integrity failure if it were to happen in the wild, but + % our security model assumes that the local store is trustworthy for local + % computation. + {ok, RetrievedMsg} = hb_cache:read(FakeID, #{ store => Store1 }), + ?assertMatch( + #{ <<"key-both">> := <<"value-both">> }, + hb_cache:ensure_all_loaded(RetrievedMsg) + ), + % Ensure that we _cannot_ read the message back from the remote node. This + % should fail despite the remote peer returning a valid message (with the + % wrong message) because the `multirequest-admissible' directive will fail. + ?assertMatch( + not_found, + hb_cache:read(FakeID, #{ store => RemoteStore }) + ). + +multiread_swapped_id_test() -> + #{ + ids_single := [ID1, ID2], + nodes := [Node1, Node2], + stores := [Store1, Store2], + remote_store := RemoteStore + } = multinode_env(), + % Link ID2 to ID1 on the first node and store. The first node will + % return ID2 but with the wrong message. It should fail and trigger a + % call to the second node, which should return it correctly. + ok = hb_store:make_link(Store1, ID1, ID2), + ?assertMatch( + {ok, #{ <<"key2">> := _ }}, + hb_cache:read(ID2, #{ store => Store2 }) + ), + ?assertMatch( + {ok, #{ <<"key1">> := _ }}, + hb_cache:read(ID2, #{ store => Store1 }) + ), + % Verify that a remote store with only the corrupt node will not return ID2, + % but a store with both the corrupt and correct nodes will. + ?assertMatch( + not_found, + hb_cache:read( + ID2, + #{ store => RemoteStore#{ <<"nodes">> => [#{ <<"prefix">> => Node1 }] } } + ) + ), + ?assertMatch( + {ok, #{ <<"key2">> := _ }}, + hb_cache:read(ID2, #{ store => Store2 }) + ), + ?assertMatch( + {ok, #{ <<"key2">> := _ }}, + hb_cache:read(ID2, #{ store => RemoteStore }) + ). + +multiread_admissible_response_hook_test() -> + #{ + ids_single := [ID1|_], + remote_store := BaseRemoteStore + } = multinode_env(), + % Ensure that we can execute a hook on valid read responses. + LogStore = [hb_test_utils:test_store()], + RemoteStore = + BaseRemoteStore#{ + <<"on">> => #{ + <<"~cache@1.0">> => + #{ + <<"admissible-response">> => #{ + <<"device">> => <<"test-device@1.0">>, + <<"store">> => LogStore, + <<"path">> => <<"log-request">> + } + } } - ), - RemoteStore = [ - #{ <<"store-module">> => hb_store_remote_node, - <<"node">> => Node, - <<"only-ids">> => true } - ], - ?assertEqual(not_found, hb_cache:read(ID, #{ store => RemoteStore })). + }, + Opts = #{ store => RemoteStore }, + ?assertMatch( + {ok, #{ <<"key1">> := _ }}, + hb_cache:read(ID1, #{ store => RemoteStore }) + ), + ?assertMatch( + {ok, Logs} when is_map(Logs) andalso map_size(Logs) > 1, + hb_ao:resolve( + #{ <<"device">> => <<"test-device@1.0">> }, + <<"logs">>, + Opts#{ store => LogStore } + ) + ). + +arweave_dot_net_as_remote_node_test() -> + TestIDs = + [ + <<"93Ui7nOLDNVCVMLeFkVeeOCVkm5Jy-kf6FNatW3q2TI">>, + <<"VuhnX2G8qVAb6kwHOiCQKl2c-42uoMKSIpHgKc0Pnzg">> + ], + Opts = + #{ + store => + [ + #{ + <<"store-module">> => hb_store_remote_node, + <<"name">> => <<"cache-arweave">>, + <<"node">> => <<"https://arweave.net">> + } + ] + }, + % Recent bundled AO messages -- no `signature` tag collision. + lists:foreach( + fun(ID) -> + {ok, M} = hb_cache:read(ID, Opts), + ?assert(hb_message:verify(M, all, Opts)) + end, + TestIDs + ). \ No newline at end of file