Merge branch 'main' into f/evercbor-warning-silence

Author: maxtropets

doc/architecture/consensus/index.rst

@@ -16,16 +16,14 @@ CFT parameters can be configured when starting up a network (see :doc:`here </op
 Extensions for Omission Faults
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-.. warning:: Support for these extensions is work-in-progress. See https://github.com/microsoft/CCF/issues/2577. 
-
 The CFT consensus implementation in CCF also supports some extensions for :term:`omission fault`.
 This may happen when the network is unreliable and may lead to one or more nodes being isolated from the rest of the network.
 
 Supported extensions include:
 
 - "CheckQuorum": the primary node automatically steps down, in the same view, if it does not hear back (via ``AppendEntriesResponse`` messages) from a majority of backups within a ``consensus.election_timeout`` period. This prevents an isolated primary node from still processing client write requests without being able to commit them.
-- "NoTimeoutRetirement": a primary node that completes its retirement sends a ProposeRequestVote message to the most up-to-date node in the new configuration, causing that node to run for election without waiting for time out.
-  - A ProposeRequestVote message is also sent when a primary receives a termination signal. This reduces downtime when the orchestrator must suddenly retire the primary's host, but there is insufficient time to reconfigure the network first.
+- "NoTimeoutRetirement": a primary node that completes its retirement sends a ``ProposeRequestVote`` message to the most up-to-date node in the new configuration, causing that node to run for election without waiting for time out.
+  - A ``ProposeRequestVote`` message is also sent when a primary receives a termination signal. This reduces downtime when the orchestrator must suddenly retire the primary's host, but there is insufficient time to reconfigure the network first.
 - "PreVote": followers must first request a pre-vote before starting a new election. This prevents followers from starting elections (and increasing the term) when they are isolated from the rest of the network.
 
 Replica State Machine

include/ccf/historical_queries_utils.h

@@ -39,7 +39,8 @@ namespace ccf::historical
   bool populate_cose_service_endorsements(
     ccf::kv::ReadOnlyTx& tx,
     ccf::historical::StatePtr& state,
-    AbstractStateCache& state_cache);
+    std::shared_ptr<NetworkIdentitySubsystemInterface>
+      network_identity_subsystem);
 
   // Verifies CCF COSE receipt using the *current network* identity's
   // certificate.

include/ccf/network_identity_interface.h

@@ -12,6 +12,16 @@ namespace ccf
 {
   struct NetworkIdentity;
 
+  using RawCoseEndorsement = std::vector<uint8_t>;
+  using CoseEndorsementsChain = std::vector<RawCoseEndorsement>;
+
+  enum class FetchStatus : uint8_t
+  {
+    Retry,
+    Done,
+    Failed
+  };
+
   class NetworkIdentitySubsystemInterface : public ccf::AbstractNodeSubSystem
   {
   public:
@@ -23,5 +33,10 @@ namespace ccf
     }
 
     virtual const std::unique_ptr<NetworkIdentity>& get() = 0;
+
+    [[nodiscard]] virtual FetchStatus endorsements_fetching_status() const = 0;
+
+    [[nodiscard]] virtual std::optional<CoseEndorsementsChain>
+    get_cose_endorsements_chain(ccf::SeqNo seqno) const = 0;
   };
 }

src/enclave/enclave.h

@@ -126,7 +126,7 @@ namespace ccf
 
       context->install_subsystem(
         std::make_shared<ccf::NetworkIdentitySubsystem>(
-          *node, network.identity));
+          *node, network.identity, historical_state_cache));
 
       context->install_subsystem(
         std::make_shared<ccf::NodeConfigurationSubsystem>(*node));

src/node/historical_queries_adapter.cpp

@@ -573,7 +573,7 @@ namespace ccf::historical
             state_cache,
             network_identity_subsystem)) ||
           !populate_cose_service_endorsements(
-            args.tx, historical_state, state_cache))
+            args.tx, historical_state, network_identity_subsystem))
         {
           auto reason = fmt::format(
             "Historical transaction {} is not currently available.",

src/node/historical_queries_utils.cpp

@@ -23,216 +23,11 @@ namespace
     bool retry{false};
   };
 
-  std::vector<ccf::CoseEndorsement> cose_endorsements_cache = {};
-
-  bool is_self_endorsement(const ccf::CoseEndorsement& endorsement)
-  {
-    return !endorsement.previous_version.has_value();
-  }
-
-  void validate_fetched_endorsement(const ccf::CoseEndorsement& endorsement)
-  {
-    if (!is_self_endorsement(endorsement))
-    {
-      const auto [from, to] =
-        ccf::crypto::extract_cose_endorsement_validity(endorsement.endorsement);
-
-      const auto from_txid = ccf::TxID::from_str(from);
-      if (!from_txid)
-      {
-        throw std::logic_error(fmt::format(
-          "Cannot parse COSE endorsement header: {}",
-          ccf::crypto::COSE_PHEADER_KEY_RANGE_BEGIN));
-      }
-
-      const auto to_txid = ccf::TxID::from_str(to);
-      if (!to_txid)
-      {
-        throw std::logic_error(fmt::format(
-          "Cannot parse COSE endorsement header: {}",
-          ccf::crypto::COSE_PHEADER_KEY_RANGE_END));
-      }
-
-      if (!endorsement.endorsement_epoch_end)
-      {
-        throw std::logic_error(
-          "COSE endorsement doesn't contain epoch end in the table entry");
-      }
-      if (
-        endorsement.endorsement_epoch_begin != *from_txid ||
-        *endorsement.endorsement_epoch_end != *to_txid)
-      {
-        throw std::logic_error(fmt ::format(
-          "COSE endorsement fetched but range is invalid, epoch begin {}, "
-          "epoch end {}, header epoch begin: {}, header epoch end: {}",
-          endorsement.endorsement_epoch_begin.to_str(),
-          endorsement.endorsement_epoch_end->to_str(),
-          from,
-          to));
-      }
-    }
-  }
-
-  void validate_chain_integrity(
-    const ccf::CoseEndorsement& newer, const ccf::CoseEndorsement& older)
-  {
-    if (
-      !is_self_endorsement(older) &&
-      (older.endorsement_epoch_end.has_value() &&
-       (newer.endorsement_epoch_begin.view - aft::starting_view_change !=
-          older.endorsement_epoch_end->view ||
-        newer.endorsement_epoch_begin.seqno - 1 !=
-          older.endorsement_epoch_end->seqno)))
-    {
-      throw std::logic_error(fmt::format(
-        "COSE endorsement chain integrity is violated, previous endorsement "
-        "epoch end {} is not chained with newer endorsement epoch begin {}",
-        older.endorsement_epoch_end->to_str(),
-        newer.endorsement_epoch_begin.to_str()));
-    }
-  }
-
-  void ensure_first_fetch(ccf::kv::ReadOnlyTx& tx)
-  {
-    if (cose_endorsements_cache.empty()) [[unlikely]]
-    {
-      const auto endorsement =
-        tx.template ro<ccf::PreviousServiceIdentityEndorsement>(
-            ccf::Tables::PREVIOUS_SERVICE_IDENTITY_ENDORSEMENT)
-          ->get();
-
-      if (!endorsement.has_value())
-      {
-        throw std::logic_error("Fetched COSE endorsement is invalid");
-      }
-
-      validate_fetched_endorsement(endorsement.value());
-
-      // Checked by the validate call above
-      cose_endorsements_cache.push_back(*endorsement);
-    }
-  }
-
   ccf::historical::CompoundHandle make_system_handle(ccf::SeqNo seq)
   {
     return ccf::historical::CompoundHandle{
       ccf::historical::RequestNamespace::System, seq};
   }
-
-  bool keep_fetching(ccf::SeqNo target_seq)
-  {
-    return !is_self_endorsement(cose_endorsements_cache.back()) &&
-      cose_endorsements_cache.back().endorsement_epoch_begin.seqno > target_seq;
-  }
-
-  FetchResult fetch_endorsements_for(
-    ccf::kv::ReadOnlyTx& tx,
-    ccf::historical::AbstractStateCache& state_cache,
-    ccf::SeqNo target_seq)
-  {
-    ensure_first_fetch(tx);
-
-    while (keep_fetching(target_seq))
-    {
-      auto& last_cose_endorsement = cose_endorsements_cache.back();
-      if (!last_cose_endorsement.previous_version.has_value())
-      {
-        throw std::logic_error(fmt::format(
-          "previous_version is not set for the endorsement with epoch_begin: "
-          "{}",
-          last_cose_endorsement.endorsement_epoch_begin.to_str()));
-      }
-      const auto prev_endorsement_seqno =
-        last_cose_endorsement.previous_version.value();
-
-      const auto system_handle = make_system_handle(prev_endorsement_seqno);
-      auto* cache_impl =
-        dynamic_cast<ccf::historical::StateCacheImpl*>(&state_cache);
-      if (cache_impl == nullptr)
-      {
-        throw std::logic_error(
-          "StateCacheImpl required to access cache as "
-          "RequestNamespace::System");
-      }
-
-      const auto hstate =
-        cache_impl->get_state_at(system_handle, prev_endorsement_seqno);
-
-      if (!hstate)
-      {
-        return {.endorsements = std::nullopt, .retry = true};
-      }
-
-      auto htx = hstate->store->create_read_only_tx();
-      const auto endorsement =
-        htx
-          .template ro<ccf::PreviousServiceIdentityEndorsement>(
-            ccf::Tables::PREVIOUS_SERVICE_IDENTITY_ENDORSEMENT)
-          ->get();
-
-      if (!endorsement.has_value())
-      {
-        throw std::logic_error("Fetched COSE endorsement is invalid");
-      }
-
-      validate_fetched_endorsement(endorsement.value());
-
-      // Checked by the validate call above
-      validate_chain_integrity(last_cose_endorsement, endorsement.value());
-      cose_endorsements_cache.push_back(endorsement.value());
-    }
-
-    if (cose_endorsements_cache.size() == 1)
-    {
-      LOG_TRACE_FMT(
-        "Only current service self-endorsement was found, no historical TXs "
-        "for previous epochs were COSE-endorsed.");
-      return {.endorsements = std::nullopt, .retry = false};
-    }
-
-    auto last_valid_endorsement = cose_endorsements_cache.end() - 1;
-    if (is_self_endorsement(*last_valid_endorsement))
-    {
-      --last_valid_endorsement;
-    }
-
-    const auto search_to = last_valid_endorsement + 1;
-
-    if (last_valid_endorsement->endorsement_epoch_begin.seqno > target_seq)
-    {
-      LOG_TRACE_FMT(
-        "COSE-endorsements are fetched for newer epochs, but target_seq {} is "
-        "far behind and was never endorsed.",
-        target_seq);
-
-      return {.endorsements = std::nullopt, .retry = false};
-    }
-
-    const auto final_endorsement = std::upper_bound(
-      cose_endorsements_cache.begin(),
-      search_to,
-      target_seq,
-      [](const auto& seq, const auto& endorsement) {
-        return endorsement.endorsement_epoch_begin.seqno <= seq;
-      });
-
-    if (final_endorsement == search_to)
-    {
-      throw std::logic_error(fmt::format(
-        "Error during COSE endorsement chain reconstruction for seqno {}",
-        target_seq));
-    }
-
-    Endorsements endorsements;
-
-    std::transform(
-      cose_endorsements_cache.begin(),
-      final_endorsement + 1, // Inclusive
-      std::back_inserter(endorsements),
-      [](const auto& e) { return e.endorsement; });
-
-    return {.endorsements = std::move(endorsements), .retry = false};
-  }
 }
 
 namespace ccf
@@ -400,40 +195,53 @@ namespace ccf
     bool populate_cose_service_endorsements(
       ccf::kv::ReadOnlyTx& tx,
       ccf::historical::StatePtr& state,
-      AbstractStateCache& state_cache)
+      std::shared_ptr<NetworkIdentitySubsystemInterface>
+        network_identity_subsystem)
     {
-      const auto service_info = tx.template ro<Service>(Tables::SERVICE)->get();
-      if (!service_info)
+      auto* service = tx.template ro<Service>(Tables::SERVICE);
+      auto hservice_info = service->get();
+      if (!hservice_info)
       {
-        throw std::logic_error(
-          "COSE endorsements fetch: current service info not available");
+        throw std::runtime_error("Failed to locate service identity");
       }
-      const auto service_start = service_info->current_service_create_txid;
-      if (!service_start)
+      if (!hservice_info->current_service_create_txid)
       {
-        throw std::logic_error(
-          "COSE endorsements fetch: current service create_txid not available");
+        throw std::runtime_error(
+          "The service identity is missing 'current_service_create_txid'");
       }
 
-      const auto target_seq = state->transaction_id.seqno;
-      if (service_start->seqno <= target_seq)
+      if (
+        state->transaction_id.seqno >=
+        hservice_info->current_service_create_txid->seqno)
       {
-        LOG_TRACE_FMT(
-          "Target seqno {} belongs to current service started at {}",
-          target_seq,
-          service_start->seqno);
+        // This is handled by the network identity subsystem, but to test
+        // mid-recovery receipts for the current service identity we set empty
+        // chain as a valid chain early on.
         return true;
       }
 
-      const auto result =
-        fetch_endorsements_for(tx, state_cache, state->transaction_id.seqno);
-      if (!result.endorsements)
+      const auto fetching =
+        network_identity_subsystem->endorsements_fetching_status();
+      if (fetching == FetchStatus::Retry)
+      {
+        return false;
+      }
+      if (fetching == FetchStatus::Failed)
+      {
+        throw std::runtime_error(fmt::format(
+          "The service identity endorsement for the receipt at seqno {} "
+          "cannot be fetched",
+          state->transaction_id.seqno));
+      }
+      if (fetching != FetchStatus::Done)
       {
-        const bool final_result = !result.retry;
-        return final_result;
+        throw std::logic_error("Unexpected endorsements fetching status");
       }
 
-      state->receipt->cose_endorsements = result.endorsements.value();
+      auto cose_endorsements =
+        network_identity_subsystem->get_cose_endorsements_chain(
+          state->transaction_id.seqno);
+      state->receipt->cose_endorsements = cose_endorsements;
       return true;
     }