backport: Merge bitcoin#28523, 27822, 28671, 29114, 28037

src/addrman.cpp

@@ -846,6 +846,30 @@ std::vector<CAddress> AddrManImpl::GetAddr_(size_t max_addresses, size_t max_pct
     return addresses;
 }
 
+std::vector<std::pair<AddrInfo, AddressPosition>> AddrManImpl::GetEntries_(bool from_tried) const
+{
+    AssertLockHeld(cs);
+
+    const int bucket_count = from_tried ? ADDRMAN_TRIED_BUCKET_COUNT : ADDRMAN_NEW_BUCKET_COUNT;
+    std::vector<std::pair<AddrInfo, AddressPosition>> infos;
+    for (int bucket = 0; bucket < bucket_count; ++bucket) {
+        for (int position = 0; position < ADDRMAN_BUCKET_SIZE; ++position) {
+            int id = GetEntry(from_tried, bucket, position);
+            if (id >= 0) {
+                AddrInfo info = mapInfo.at(id);
+                AddressPosition location = AddressPosition(
+                    from_tried,
+                    /*multiplicity_in=*/from_tried ? 1 : info.nRefCount,
+                    bucket,
+                    position);
+                infos.push_back(std::make_pair(info, location));
+            }
+        }
+    }
+
+    return infos;
+}
+
 void AddrManImpl::Connected_(const CService& addr, NodeSeconds time)
 {
     AssertLockHeld(cs);
@@ -1218,6 +1242,15 @@ std::vector<CAddress> AddrManImpl::GetAddr(size_t max_addresses, size_t max_pct,
     return addresses;
 }
 
+std::vector<std::pair<AddrInfo, AddressPosition>> AddrManImpl::GetEntries(bool from_tried) const
+{
+    LOCK(cs);
+    Check();
+    auto addrInfos = GetEntries_(from_tried);
+    Check();
+    return addrInfos;
+}
+
 void AddrManImpl::Connected(const CService& addr, NodeSeconds time)
 {
     LOCK(cs);
@@ -1320,6 +1353,11 @@ std::vector<CAddress> AddrMan::GetAddr(size_t max_addresses, size_t max_pct, std
     return m_impl->GetAddr(max_addresses, max_pct, network, filtered);
 }
 
+std::vector<std::pair<AddrInfo, AddressPosition>> AddrMan::GetEntries(bool use_tried) const
+{
+    return m_impl->GetEntries(use_tried);
+}
+
 void AddrMan::Connected(const CService& addr, NodeSeconds time)
 {
     m_impl->Connected(addr, time);

src/addrman.h

@@ -40,7 +40,7 @@ class AddrManImpl;
 /** Default for -checkaddrman */
 static constexpr int32_t DEFAULT_ADDRMAN_CONSISTENCY_CHECKS{0};
 
-/** Test-only struct, capturing info about an address in AddrMan */
+/** Location information for an address in AddrMan */
 struct AddressPosition {
     // Whether the address is in the new or tried table
     const bool tried;
@@ -183,6 +183,17 @@ class AddrMan
      */
     std::vector<CAddress> GetAddr(size_t max_addresses, size_t max_pct, std::optional<Network> network, const bool filtered = true) const;
 
+    /**
+     * Returns an information-location pair for all addresses in the selected addrman table.
+     * If an address appears multiple times in the new table, an information-location pair
+     * is returned for each occurence. Addresses only ever appear once in the tried table.
+     *
+     * @param[in] from_tried     Selects which table to return entries from.
+     *
+     * @return                   A vector consisting of pairs of AddrInfo and AddressPosition.
+     */
+    std::vector<std::pair<AddrInfo, AddressPosition>> GetEntries(bool from_tried) const;
+
     /** We have successfully connected to this peer. Calling this function
      *  updates the CAddress's nTime, which is used in our IsTerrible()
      *  decisions and gossiped to peers. Callers should be careful that updating

src/addrman_impl.h

@@ -133,6 +133,9 @@ class AddrManImpl
     std::vector<CAddress> GetAddr(size_t max_addresses, size_t max_pct, std::optional<Network> network, const bool filtered = true) const
         EXCLUSIVE_LOCKS_REQUIRED(!cs);
 
+    std::vector<std::pair<AddrInfo, AddressPosition>> GetEntries(bool from_tried) const
+        EXCLUSIVE_LOCKS_REQUIRED(!cs);
+
     void Connected(const CService& addr, NodeSeconds time)
         EXCLUSIVE_LOCKS_REQUIRED(!cs);
 
@@ -264,6 +267,8 @@ class AddrManImpl
 
     std::vector<CAddress> GetAddr_(size_t max_addresses, size_t max_pct, std::optional<Network> network, const bool filtered = true) const EXCLUSIVE_LOCKS_REQUIRED(cs);
 
+    std::vector<std::pair<AddrInfo, AddressPosition>> GetEntries_(bool from_tried) const EXCLUSIVE_LOCKS_REQUIRED(cs);
+
     void Connected_(const CService& addr, NodeSeconds time) EXCLUSIVE_LOCKS_REQUIRED(cs);
 
     void SetServices_(const CService& addr, ServiceFlags nServices) EXCLUSIVE_LOCKS_REQUIRED(cs);

src/rpc/client.cpp

@@ -458,10 +458,10 @@ UniValue RPCConvertNamedValues(std::string strMethod, const std::vector<std::str
     }
 
     if (!positional_args.empty()) {
-        // Use __pushKV instead of pushKV to avoid overwriting an explicit
+        // Use pushKVEnd instead of pushKV to avoid overwriting an explicit
         // "args" value with an implicit one. Let the RPC server handle the
         // request as given.
-        params.__pushKV("args", positional_args);
+        params.pushKVEnd("args", positional_args);
     }
 
     return params;

src/rpc/mempool.cpp

@@ -346,8 +346,8 @@ UniValue MempoolToJSON(const CTxMemPool& pool, const llmq::CInstantSendManager*
             entryToJSON(pool, info, e, isman);
             // Mempool has unique entries so there is no advantage in using
             // UniValue::pushKV, which checks if the key already exists in O(N).
-            // UniValue::__pushKV is used instead which currently is O(1).
-            o.__pushKV(hash.ToString(), info);
+            // UniValue::pushKVEnd is used instead which currently is O(1).
+            o.pushKVEnd(hash.ToString(), info);
         }
         return o;
     } else {

src/rpc/net.cpp

@@ -6,6 +6,7 @@
 #include <rpc/server.h>
 
 #include <addrman.h>
+#include <addrman_impl.h>
 #include <banman.h>
 #include <chainparams.h>
 #include <clientversion.h>
@@ -1174,6 +1175,74 @@ static RPCHelpMan getaddrmaninfo()
     };
 }
 
+UniValue AddrmanEntryToJSON(const AddrInfo& info)
+{
+    UniValue ret(UniValue::VOBJ);
+    ret.pushKV("address", info.ToStringAddr());
+    ret.pushKV("port", info.GetPort());
+    ret.pushKV("services", (uint64_t)info.nServices);
+    ret.pushKV("time", int64_t{TicksSinceEpoch<std::chrono::seconds>(info.nTime)});
+    ret.pushKV("network", GetNetworkName(info.GetNetClass()));
+    ret.pushKV("source", info.source.ToStringAddr());
+    ret.pushKV("source_network", GetNetworkName(info.source.GetNetClass()));
+    return ret;
+}
+
+UniValue AddrmanTableToJSON(const std::vector<std::pair<AddrInfo, AddressPosition>>& tableInfos)
+{
+    UniValue table(UniValue::VOBJ);
+    for (const auto& e : tableInfos) {
+        AddrInfo info = e.first;
+        AddressPosition location = e.second;
+        std::ostringstream key;
+        key << location.bucket << "/" << location.position;
+        // Address manager tables have unique entries so there is no advantage
+        // in using UniValue::pushKV, which checks if the key already exists
+        // in O(N). UniValue::pushKVEnd is used instead which currently is O(1).
+        table.pushKVEnd(key.str(), AddrmanEntryToJSON(info));
+    }
+    return table;
+}
+
+static RPCHelpMan getrawaddrman()
+{
+    return RPCHelpMan{"getrawaddrman",
+        "EXPERIMENTAL warning: this call may be changed in future releases.\n"
+        "\nReturns information on all address manager entries for the new and tried tables.\n",
+        {},
+        RPCResult{
+            RPCResult::Type::OBJ_DYN, "", "", {
+                {RPCResult::Type::OBJ_DYN, "table", "buckets with addresses in the address manager table ( new, tried )", {
+                    {RPCResult::Type::OBJ, "bucket/position", "the location in the address manager table (<bucket>/<position>)", {
+                        {RPCResult::Type::STR, "address", "The address of the node"},
+                        {RPCResult::Type::NUM, "port", "The port number of the node"},
+                        {RPCResult::Type::STR, "network", "The network (" + Join(GetNetworkNames(), ", ") + ") of the address"},
+                        {RPCResult::Type::NUM, "services", "The services offered by the node"},
+                        {RPCResult::Type::NUM_TIME, "time", "The " + UNIX_EPOCH_TIME + " when the node was last seen"},
+                        {RPCResult::Type::STR, "source", "The address that relayed the address to us"},
+                        {RPCResult::Type::STR, "source_network", "The network (" + Join(GetNetworkNames(), ", ") + ") of the source address"},
+                    }}
+                }}
+            }
+        },
+        RPCExamples{
+            HelpExampleCli("getrawaddrman", "")
+            + HelpExampleRpc("getrawaddrman", "")
+        },
+        [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue {
+            NodeContext& node = EnsureAnyNodeContext(request.context);
+            if (!node.addrman) {
+                throw JSONRPCError(RPC_CLIENT_P2P_DISABLED, "Error: Address manager functionality missing or disabled");
+            }
+
+            UniValue ret(UniValue::VOBJ);
+            ret.pushKV("new", AddrmanTableToJSON(node.addrman->GetEntries(false)));
+            ret.pushKV("tried", AddrmanTableToJSON(node.addrman->GetEntries(true)));
+            return ret;
+        },
+    };
+}
+
 void RegisterNetRPCCommands(CRPCTable &t)
 {
     static const CRPCCommand commands[]{
@@ -1196,6 +1265,7 @@ void RegisterNetRPCCommands(CRPCTable &t)
         {"hidden", &sendmsgtopeer},
         {"hidden", &setmnthreadactive},
         {"hidden", &getaddrmaninfo},
+        {"hidden", &getrawaddrman},
     };
     for (const auto& c : commands) {
         t.appendCommand(c.name, &c);

src/serialize.h

@@ -827,14 +827,12 @@ template<typename Stream, typename A, typename B, typename C> void Unserialize(S
 
 /**
  * prevector
- * prevectors of unsigned char are a special case and are intended to be serialized as a single opaque blob.
  */
 template<typename Stream, unsigned int N, typename T> inline void Serialize(Stream& os, const prevector<N, T>& v);
 template<typename Stream, unsigned int N, typename T> inline void Unserialize(Stream& is, prevector<N, T>& v);
 
 /**
  * vector
- * vectors of unsigned char are a special case and are intended to be serialized as a single opaque blob.
  */
 template<typename Stream, typename T, typename A> inline void Serialize(Stream& os, const std::vector<T, A>& v);
 template<typename Stream, typename T, typename A> inline void Unserialize(Stream& is, std::vector<T, A>& v);
@@ -1002,10 +1000,9 @@ void Unserialize(Stream& is, std::basic_string_view<C>& str)
 template <typename Stream, unsigned int N, typename T>
 void Serialize(Stream& os, const prevector<N, T>& v)
 {
-    if constexpr (std::is_same_v<T, unsigned char>) {
+    if constexpr (BasicByte<T>) { // Use optimized version for unformatted basic bytes
         WriteCompactSize(os, v.size());
-        if (!v.empty())
-            os.write(MakeByteSpan(v));
+        if (!v.empty()) os.write(MakeByteSpan(v));
     } else {
         Serialize(os, Using<VectorFormatter<DefaultFormatter>>(v));
     }
@@ -1015,7 +1012,7 @@ void Serialize(Stream& os, const prevector<N, T>& v)
 template <typename Stream, unsigned int N, typename T>
 void Unserialize(Stream& is, prevector<N, T>& v)
 {
-    if constexpr (std::is_same_v<T, unsigned char>) {
+    if constexpr (BasicByte<T>) { // Use optimized version for unformatted basic bytes
         // Limit size per read so bogus size value won't cause out of memory
         v.clear();
         unsigned int nSize = ReadCompactSize(is);
@@ -1038,10 +1035,9 @@ void Unserialize(Stream& is, prevector<N, T>& v)
 template <typename Stream, typename T, typename A>
 void Serialize(Stream& os, const std::vector<T, A>& v)
 {
-    if constexpr (std::is_same_v<T, unsigned char>) {
+    if constexpr (BasicByte<T>) { // Use optimized version for unformatted basic bytes
         WriteCompactSize(os, v.size());
-        if (!v.empty())
-            os.write(MakeByteSpan(v));
+        if (!v.empty()) os.write(MakeByteSpan(v));
     } else if constexpr (std::is_same_v<T, bool>) {
         // A special case for std::vector<bool>, as dereferencing
         // std::vector<bool>::const_iterator does not result in a const bool&
@@ -1059,7 +1055,7 @@ void Serialize(Stream& os, const std::vector<T, A>& v)
 template <typename Stream, typename T, typename A>
 void Unserialize(Stream& is, std::vector<T, A>& v)
 {
-    if constexpr (std::is_same_v<T, unsigned char>) {
+    if constexpr (BasicByte<T>) { // Use optimized version for unformatted basic bytes
         // Limit size per read so bogus size value won't cause out of memory
         v.clear();
         unsigned int nSize = ReadCompactSize(is);

src/span.h

@@ -270,9 +270,11 @@ Span<std::byte> MakeWritableByteSpan(V&& v) noexcept
 // Helper functions to safely cast basic byte pointers to unsigned char pointers.
 inline unsigned char* UCharCast(char* c) { return reinterpret_cast<unsigned char*>(c); }
 inline unsigned char* UCharCast(unsigned char* c) { return c; }
+inline unsigned char* UCharCast(signed char* c) { return reinterpret_cast<unsigned char*>(c); }
 inline unsigned char* UCharCast(std::byte* c) { return reinterpret_cast<unsigned char*>(c); }
 inline const unsigned char* UCharCast(const char* c) { return reinterpret_cast<const unsigned char*>(c); }
 inline const unsigned char* UCharCast(const unsigned char* c) { return c; }
+inline const unsigned char* UCharCast(const signed char* c) { return reinterpret_cast<const unsigned char*>(c); }
 inline const unsigned char* UCharCast(const std::byte* c) { return reinterpret_cast<const unsigned char*>(c); }
 // Helper concept for the basic byte types.
 template <typename B>

src/test/fuzz/rpc.cpp

@@ -135,6 +135,7 @@ const std::vector<std::string> RPC_COMMANDS_SAFE_FOR_FUZZING{
     "getnetworkinfo",
     "getnodeaddresses",
     "getpeerinfo",
+    "getrawaddrman",
     "getrawmempool",
     "getrawtransaction",
     "getrpcinfo",

src/test/settings_tests.cpp

@@ -35,7 +35,7 @@ inline std::ostream& operator<<(std::ostream& os, const util::SettingsValue& val
 inline std::ostream& operator<<(std::ostream& os, const std::pair<std::string, util::SettingsValue>& kv)
 {
     util::SettingsValue out(util::SettingsValue::VOBJ);
-    out.__pushKV(kv.first, kv.second);
+    out.pushKVEnd(kv.first, kv.second);
     os << out.write();
     return os;
 }

src/txorphanage.cpp

@@ -59,10 +59,10 @@ bool TxOrphanage::AddTx(const CTransactionRef& tx, NodeId peer)
 int TxOrphanage::EraseTx(const uint256& txid)
 {
     LOCK(m_mutex);
-    return _EraseTx(txid);
+    return EraseTxNoLock(txid);
 }
 
-int TxOrphanage::_EraseTx(const uint256& txid)
+int TxOrphanage::EraseTxNoLock(const uint256& txid)
 {
     AssertLockHeld(m_mutex);
     std::map<uint256, OrphanTx>::iterator it = m_orphans.find(txid);
@@ -110,7 +110,7 @@ void TxOrphanage::EraseForPeer(NodeId peer)
         std::map<uint256, OrphanTx>::iterator maybeErase = iter++; // increment to avoid iterator becoming invalid
         if (maybeErase->second.fromPeer == peer)
         {
-            nErased += _EraseTx(maybeErase->second.tx->GetHash());
+            nErased += EraseTxNoLock(maybeErase->second.tx->GetHash());
         }
     }
     if (nErased > 0) LogPrint(BCLog::MEMPOOL, "Erased %d orphan tx from peer=%d\n", nErased, peer);
@@ -132,7 +132,7 @@ void TxOrphanage::LimitOrphans(unsigned int max_orphans_size)
         {
             std::map<uint256, OrphanTx>::iterator maybeErase = iter++;
             if (maybeErase->second.nTimeExpire <= nNow) {
-                nErased += _EraseTx(maybeErase->second.tx->GetHash());
+                nErased += EraseTxNoLock(maybeErase->second.tx->GetHash());
             } else {
                 nMinExpTime = std::min(maybeErase->second.nTimeExpire, nMinExpTime);
             }
@@ -146,7 +146,7 @@ void TxOrphanage::LimitOrphans(unsigned int max_orphans_size)
     {
         // Evict a random orphan:
         size_t randompos = rng.randrange(m_orphan_list.size());
-        _EraseTx(m_orphan_list[randompos]->first);
+        EraseTxNoLock(m_orphan_list[randompos]->first);
         ++nEvicted;
     }
     if (nEvicted > 0) LogPrint(BCLog::MEMPOOL, "orphanage overflow, removed %u tx\n", nEvicted);
@@ -234,7 +234,7 @@ void TxOrphanage::EraseForBlock(const CBlock& block)
     if (vOrphanErase.size()) {
         int nErased = 0;
         for (const uint256& orphanHash : vOrphanErase) {
-            nErased += _EraseTx(orphanHash);
+            nErased += EraseTxNoLock(orphanHash);
         }
         LogPrint(BCLog::MEMPOOL, "Erased %d orphan tx included or conflicted by block\n", nErased);
     }

src/txorphanage.h

@@ -104,7 +104,7 @@ class TxOrphanage {
     size_t m_orphan_tx_size GUARDED_BY(m_mutex){0};
 
     /** Erase an orphan by txid */
-    int _EraseTx(const uint256& txid) EXCLUSIVE_LOCKS_REQUIRED(m_mutex);
+    int EraseTxNoLock(const uint256& txid) EXCLUSIVE_LOCKS_REQUIRED(m_mutex);
 };
 
 #endif // BITCOIN_TXORPHANAGE_H

src/univalue/include/univalue.h

@@ -87,7 +87,7 @@ class UniValue {
     template <class It>
     void push_backV(It first, It last);
 
-    void __pushKV(std::string key, UniValue val);
+    void pushKVEnd(std::string key, UniValue val);
     void pushKV(std::string key, UniValue val);
     void pushKVs(UniValue obj);