DO NOT MERGE: Update to crypton == 1.1 and replace memory package with ram

cabal.project

@@ -48,11 +48,6 @@ jobs: $ncpus
 semaphore: True
 
 
-constraints:
-  -- haskell.nix patch does not work for 1.6.8
-  , any.crypton-x509-system < 1.6.8
-
-
 -- WASM compilation specific
 
 if arch(wasm32)
@@ -166,3 +161,97 @@ if impl(ghc >=9.14)
     , time-locale-compat:time
     , with-utf8:base
 -- cabal-allow-newer end
+
+-- TEMPORARY: switching to crypton >= 1.1 (which depends on `ram` instead of
+-- `memory`) needs cardano-crypto-class-2.6.0.0 (the first version depending
+-- on crypton ^>=1.1 and `ram`), but the released ouroboros-consensus-4.1.0.0
+-- pins `cardano-crypto-class ^>=2.5`, which excludes it. The blocks below
+-- replicate the (draft, do-not-merge) upstream fix in
+-- https://github.com/IntersectMBO/ouroboros-consensus/pull/2213, which widens
+-- ouroboros-consensus's bound to also allow cardano-crypto-class-2.6.0.0 and
+-- pulls in the handful of other packages that need to move in lockstep with
+-- it. Remove all of this once these land upstream and a release picks them up.
+allow-newer:
+  cardano-ledger-mary:cardano-crypto-class,
+  cardano-ledger-shelley:cardano-crypto-class,
+  cardano-ledger-binary:cardano-crypto-class,
+  cardano-ledger-core:cardano-crypto-class,
+  cardano-protocol:cardano-crypto-class,
+  cardano-protocol-tpraos:cardano-crypto-class,
+  kes-agent:cardano-crypto-class,
+  kes-agent-crypto:cardano-crypto-class,
+
+source-repository-package
+  type: git
+  location: https://github.com/IntersectMBO/ouroboros-consensus
+  tag: 206dffef96e76e42e15a1fb9fd6796394aef3409
+
+-- TEMPORARY: pulls in cardano-base PR #694 (erikd/contra-tracer, not yet
+-- merged) https://github.com/IntersectMBO/cardano-base/pull/694, which widens
+-- cardano-crypto-class:testlib's contra-tracer bound so it builds against the
+-- same contra-tracer version as everything else here. Needed by the
+-- ouroboros-consensus pin above. Remove once this PR (or an equivalent fix)
+-- is released.
+source-repository-package
+  type: git
+  location: https://github.com/IntersectMBO/cardano-base
+  tag: ad3afec8113872652ec8edf19ae1280552ce2a20
+  subdir: cardano-crypto-class
+
+-- TEMPORARY: pulls in kes-agent PR #105 (erikd/contra-tracer, not yet merged)
+-- https://github.com/input-output-hk/kes-agent/pull/105, which widens the
+-- same contra-tracer bound for kes-agent/kes-agent-crypto, on top of the
+-- FixedSizeCodec migration cardano-crypto-class-2.6.0.0 needs. Needed by the
+-- ouroboros-consensus pin above. Remove once this PR (or an equivalent fix)
+-- is released.
+source-repository-package
+  type: git
+  location: https://github.com/input-output-hk/kes-agent
+  tag: 0e9a16c61ecc6b5cc747ae0ebdb718a02eae8ddc
+  subdir:
+    kes-agent
+    kes-agent-crypto
+
+-- TEMPORARY: pulls in cardano-ledger PR #5999 (erikd/ram, not yet merged)
+-- https://github.com/IntersectMBO/cardano-ledger/pull/5999, which switches
+-- cardano-crypto-wrapper from `memory` to `ram` so it builds against crypton
+-- >=1.1. Needed by the ouroboros-consensus pin above. Remove once this PR (or
+-- an equivalent fix) is released.
+source-repository-package
+  type: git
+  location: https://github.com/IntersectMBO/cardano-ledger
+  tag: 407b27ad62459cccf346145f97f63a259ae7ec75
+  subdir: eras/byron/crypto
+
+-- TEMPORARY: cardano-addresses-4.0.2 (the latest released on CHaP) still pins
+-- `crypton >=0.32 && <1.1` and depends on the standalone `cardano-crypto`
+-- package (capped at <1.4.0, i.e. before its own `memory`->`ram` switch).
+-- master (unreleased, 4.0.7) has already moved to `crypton >=1.1 && <1.2` /
+-- `ram` and dropped the `cardano-crypto` dependency entirely. Remove once a
+-- release picks this up.
+source-repository-package
+  type: git
+  location: https://github.com/IntersectMBO/cardano-addresses
+  tag: 63c2497f965e96610c6e3df1127dbd0b9481228f
+
+-- TEMPORARY: grapesy-1.1.1 (the latest released on Hackage) pins
+-- `tls >=1.7 && <2.2`, which excludes tls-2.4.3 (needed for crypton >=1.1,
+-- since older tls versions pull in crypton-x509-validation <1.7, which caps
+-- crypton <1.1). well-typed/grapesy's master widens that to
+-- `tls >=1.7 && <2.5`, still version 1.1.1 (unreleased bump). Remove once a
+-- release picks this up.
+source-repository-package
+  type: git
+  location: https://github.com/well-typed/grapesy
+  tag: bd6af64f69ff89e3a8fc02e2c81262e648f4715d
+  subdir:
+    grapesy
+    grpc-spec
+
+-- TEMPORARY: this pin (working around a haskell.nix patch that doesn't apply
+-- to crypton-x509-system-1.6.8) forces an old crypton-x509-system, which
+-- conflicts with the crypton >=1.1 migration above (that needs
+-- crypton-x509-system-1.9.0). Dropped for now; revisit once the crypton
+-- migration lands and re-check whether the haskell.nix issue still applies.
+-- constraints:
+--   , any.crypton-x509-system < 1.6.8

cardano-api/cardano-api.cabal

@@ -133,10 +133,10 @@ library
     bytestring-trie,
     cardano-addresses ^>=4.0.0,
     cardano-base ^>=0.1,
-    cardano-binary,
+    cardano-binary >=1.9.1,
     cardano-crypto,
-    cardano-crypto-class ^>=2.5,
-    cardano-crypto-wrapper ^>=1.7,
+    cardano-crypto-class ^>=2.6,
+    cardano-crypto-wrapper ^>=1.7 || ^>=1.8,
     cardano-data >=1.0,
     cardano-diffusion:{api, cardano-diffusion} ^>=1.1,
     cardano-ledger-allegra >=1.7,
@@ -168,7 +168,6 @@ library
     formatting,
     fs-api ^>=0.4,
     iproute,
-    memory,
     mempack,
     microlens <0.6,
     mono-traversable,
@@ -186,6 +185,7 @@ library
     prettyprinter,
     prettyprinter-ansi-terminal,
     prettyprinter-configurable ^>=1.36,
+    ram,
     random,
     resource-registry ^>=0.3,
     safe-exceptions,
@@ -201,7 +201,7 @@ library
     transformers,
     transformers-except ^>=0.1.3,
     typed-protocols ^>=1.2,
-    validation,
+    validation ^>=1.2,
     vector,
     yaml,
 
@@ -329,8 +329,8 @@ library gen
     bytestring,
     cardano-api,
     cardano-binary >=1.6 && <1.10,
-    cardano-crypto-class ^>=2.5,
-    cardano-crypto-wrapper:testlib ^>=1.7,
+    cardano-crypto-class ^>=2.5 || ^>=2.6,
+    cardano-crypto-wrapper:testlib ^>=1.7 || ^>=1.8,
     cardano-ledger-alonzo:{cardano-ledger-alonzo, testlib},
     cardano-ledger-byron:testlib,
     cardano-ledger-conway:testlib,
@@ -373,7 +373,7 @@ test-suite cardano-api-test
     cardano-api:gen,
     cardano-binary,
     cardano-crypto,
-    cardano-crypto-class:{cardano-crypto-class, testlib} ^>=2.5,
+    cardano-crypto-class:{cardano-crypto-class, testlib} ^>=2.5 || ^>=2.6,
     cardano-crypto-wrapper:testlib,
     cardano-ledger-alonzo,
     cardano-ledger-api ^>=1.14,

cardano-api/src/Cardano/Api/Crypto/Ed25519Bip32.hs

@@ -40,15 +40,6 @@ data Ed25519Bip32DSIGN
 instance DSIGNAlgorithm Ed25519Bip32DSIGN where
   type SeedSizeDSIGN Ed25519Bip32DSIGN = 32
 
-  -- \| BIP32-Ed25519 extended verification key size is 64 octets.
-  type VerKeySizeDSIGN Ed25519Bip32DSIGN = 64
-
-  -- \| BIP32-Ed25519 extended signing key size is 96 octets.
-  type SignKeySizeDSIGN Ed25519Bip32DSIGN = 96
-
-  -- \| BIP32-Ed25519 extended signature size is 64 octets.
-  type SigSizeDSIGN Ed25519Bip32DSIGN = 64
-
   --
   -- Key and signature types
   --
@@ -102,21 +93,33 @@ instance DSIGNAlgorithm Ed25519Bip32DSIGN where
         (mempty :: ScrubbedBytes)
         (mempty :: ScrubbedBytes)
 
+--
+-- raw serialise/deserialise, in fixed-size raw format
+--
+
 instance FixedSizeCodec (VerKeyDSIGN Ed25519Bip32DSIGN) where
+  -- \| BIP32-Ed25519 extended verification key size is 64 octets.
   type FixedSize (VerKeyDSIGN Ed25519Bip32DSIGN) = 64
+
   rawEncodeFixedSized (VerKeyEd25519Bip32DSIGN vk) = CC.unXPub vk
   rawDecodeFixedSized bs =
     either fail (pure . VerKeyEd25519Bip32DSIGN) (CC.xpub bs)
 
 instance FixedSizeCodec (SignKeyDSIGN Ed25519Bip32DSIGN) where
+  -- \| BIP32-Ed25519 extended signing key size is 96 octets.
   type FixedSize (SignKeyDSIGN Ed25519Bip32DSIGN) = 96
+
   rawEncodeFixedSized (SignKeyEd25519Bip32DSIGN sk) = xPrvToBytes sk
   rawDecodeFixedSized bs =
-    maybe (fail "invalid Ed25519Bip32DSIGN signing key") (pure . SignKeyEd25519Bip32DSIGN) $
-      xPrvFromBytes bs
+    maybe
+      (fail "Ed25519Bip32DSIGN: invalid SignKeyDSIGN")
+      (pure . SignKeyEd25519Bip32DSIGN)
+      (xPrvFromBytes bs)
 
 instance FixedSizeCodec (SigDSIGN Ed25519Bip32DSIGN) where
+  -- \| BIP32-Ed25519 extended signature size is 64 octets.
   type FixedSize (SigDSIGN Ed25519Bip32DSIGN) = 64
+
   rawEncodeFixedSized = BA.convert
   rawDecodeFixedSized bs =
     either fail (pure . SigEd25519Bip32DSIGN) (CC.xsignature bs)

cardano-api/src/Cardano/Api/Key/Internal/Leios.hs

@@ -191,6 +191,17 @@ blsPossessionProof hexBs =
     Left e -> error $ "blsPossessionProof: " ++ show e
     Right p -> p
 
+-- | Signing context including the Domain Separation Tag (DST) for the proofs-of-possession of
+-- BLS keys using the minimal-signature-size BLS12-381 variant.
+--
+-- A Domain Separation Tag is a unique tag (like a magic number) that we add to ensure that
+-- the signature is used only in the context that it was intended for.
+-- This is because BLS keys and signatures can be used for multiple purposes, and
+-- we don't want a proof of possession for one purpose to be interpreted as something different
+-- in a different context.
+minSigPoPContext :: Crypto.BLS12381SignContext
+minSigPoPContext = Crypto.minSigPoPDST
+
 -- | Create a proof of possession for a BLS signing key.
 --
 -- This proof demonstrates that the holder of a BLS verification key knows the corresponding
@@ -199,7 +210,7 @@ blsPossessionProof hexBs =
 -- honest participants' keys during aggregation (a rogue key attack).
 createBlsPossessionProof :: SigningKey BlsKey -> BlsPossessionProof
 createBlsPossessionProof (BlsSigningKey sk) =
-  BlsPossessionProof (Crypto.createPossessionProofDSIGN Crypto.minSigPoPDST sk)
+  BlsPossessionProof (Crypto.createPossessionProofDSIGN minSigPoPContext sk)
 
 instance HasTypeProxy BlsPossessionProof where
   data AsType BlsPossessionProof = AsBlsPossessionProof

cardano-api/src/Cardano/Api/Key/Internal/Mnemonic.hs

@@ -15,6 +15,7 @@ module Cardano.Api.Key.Internal.Mnemonic
   )
 where
 
+import Cardano.Api.Crypto.Ed25519Bip32 (xPrvFromBytes)
 import Cardano.Api.Error (Error (..))
 import Cardano.Api.Key.Internal
   ( AsType
@@ -35,6 +36,7 @@ import Cardano.Address.Derivation
   , XPrv
   , genMasterKeyFromMnemonic
   , indexFromWord32
+  , xprvToBytes
   )
 import Cardano.Address.Style.Shelley
   ( Role (..)
@@ -44,6 +46,7 @@ import Cardano.Address.Style.Shelley
   , deriveDRepPrivateKey
   )
 import Cardano.Crypto.Encoding.BIP39 (Dictionary (dictionaryIndexToWord))
+import Cardano.Crypto.Wallet qualified as Crypto.HD
 import Cardano.Mnemonic
   ( MkSomeMnemonic (mkSomeMnemonic)
   , MkSomeMnemonicError (..)
@@ -59,6 +62,7 @@ import Data.ByteString qualified as BS
 import Data.Either.Combinators (mapLeft, maybeToRight)
 import Data.Either.Extra (maybeToEither)
 import Data.Foldable (toList)
+import Data.Maybe (fromMaybe)
 import Data.Text (Text)
 import Data.Text qualified as Text
 import Data.Text.Encoding (decodeUtf8, encodeUtf8)
@@ -146,6 +150,23 @@ class IndexedSigningKeyFromRootKey keyrole where
     -> Either Word32 (SigningKey keyrole)
     -- ^ The derived extended signing key or the 'indexType' if it is invalid.
 
+-- | cardano-addresses' own 'XPrv' (from key derivation) and the 'Crypto.HD.XPrv'
+-- ("Cardano.Crypto.Wallet", from the standalone @cardano-crypto@ package) that
+-- 'SigningKey' constructors expect used to be the same type: cardano-addresses
+-- re-exported cardano-crypto's. Since cardano-addresses vendored its own copy of
+-- "Cardano.Crypto.Wallet" (to drop the @cardano-crypto@/@memory@ dependency in
+-- favour of @crypton@/@ram@), the two are now distinct types, so bridge between
+-- them via the same 96-byte compact raw format both sides already use (compare
+-- 'Cardano.Address.Derivation.xprvToBytes' with
+-- 'Cardano.Api.Crypto.Ed25519Bip32.xPrvToBytes'/'xPrvFromBytes'). Both are
+-- implementations of the same BIP32-Ed25519 extended-key format, so this
+-- round-trip is lossless.
+toCryptoXPrv :: XPrv -> Crypto.HD.XPrv
+toCryptoXPrv =
+  fromMaybe (error "toCryptoXPrv: impossible: cardano-addresses' XPrv is always 96 bytes")
+    . xPrvFromBytes
+    . xprvToBytes
+
 instance IndexedSigningKeyFromRootKey PaymentExtendedKey where
   deriveSigningKeyFromAccountWithPaymentKeyIndex
     :: AsType PaymentExtendedKey
@@ -154,7 +175,11 @@ instance IndexedSigningKeyFromRootKey PaymentExtendedKey where
     -> Either Word32 (SigningKey PaymentExtendedKey)
   deriveSigningKeyFromAccountWithPaymentKeyIndex _ accK idx = do
     payKeyIx <- maybeToEither idx $ indexFromWord32 @(Index 'Soft 'PaymentK) idx
-    return $ PaymentExtendedSigningKey $ getKey $ deriveAddressPrivateKey accK UTxOExternal payKeyIx
+    return $
+      PaymentExtendedSigningKey $
+        toCryptoXPrv $
+          getKey $
+            deriveAddressPrivateKey accK UTxOExternal payKeyIx
 
 instance IndexedSigningKeyFromRootKey StakeExtendedKey where
   deriveSigningKeyFromAccountWithPaymentKeyIndex
@@ -164,31 +189,35 @@ instance IndexedSigningKeyFromRootKey StakeExtendedKey where
     -> Either Word32 (SigningKey StakeExtendedKey)
   deriveSigningKeyFromAccountWithPaymentKeyIndex _ accK idx = do
     payKeyIx <- maybeToEither idx $ indexFromWord32 @(Index 'Soft 'PaymentK) idx
-    return $ StakeExtendedSigningKey $ getKey $ deriveAddressPrivateKey accK Stake payKeyIx
+    return $
+      StakeExtendedSigningKey $
+        toCryptoXPrv $
+          getKey $
+            deriveAddressPrivateKey accK Stake payKeyIx
 
 instance SigningKeyFromRootKey DRepExtendedKey where
   deriveSigningKeyFromAccount
     :: AsType DRepExtendedKey
     -> Shelley 'AccountK XPrv
     -> SigningKey DRepExtendedKey
   deriveSigningKeyFromAccount _ accK =
-    DRepExtendedSigningKey $ getKey $ deriveDRepPrivateKey accK
+    DRepExtendedSigningKey $ toCryptoXPrv $ getKey $ deriveDRepPrivateKey accK
 
 instance SigningKeyFromRootKey CommitteeColdExtendedKey where
   deriveSigningKeyFromAccount
     :: AsType CommitteeColdExtendedKey
     -> Shelley 'AccountK XPrv
     -> SigningKey CommitteeColdExtendedKey
   deriveSigningKeyFromAccount _ accK =
-    CommitteeColdExtendedSigningKey $ getKey $ deriveCCColdPrivateKey accK
+    CommitteeColdExtendedSigningKey $ toCryptoXPrv $ getKey $ deriveCCColdPrivateKey accK
 
 instance SigningKeyFromRootKey CommitteeHotExtendedKey where
   deriveSigningKeyFromAccount
     :: AsType CommitteeHotExtendedKey
     -> Shelley 'AccountK XPrv
     -> SigningKey CommitteeHotExtendedKey
   deriveSigningKeyFromAccount _ accK =
-    CommitteeHotExtendedSigningKey $ getKey $ deriveCCHotPrivateKey accK
+    CommitteeHotExtendedSigningKey $ toCryptoXPrv $ getKey $ deriveCCHotPrivateKey accK
 
 -- | Generate a signing key from a mnemonic sentence given a function that
 -- derives a key from an account extended key.

cardano-api/src/Cardano/Api/Tx/Internal/Sign.hs

@@ -782,20 +782,24 @@ decodeShelleyBasedWitness
   -> ByteString
   -> Either CBOR.DecoderError (KeyWitness era)
 decodeShelleyBasedWitness sbe bs =
-  let e =
-        Valid.foldValidation Left Right $
-          mconcat $
-            map
-              (either (Valid.Failure . (: [])) Valid.Success)
-              [ bootstrapWitnessDecoder bs
-              , shelleyKeyWitnessDecoder bs
-              , legacyKeyWitnessDecoder bs
-              ]
-   in case e of
-        Left errs ->
-          let allErrs = Text.unlines $ map renderBuildable errs
-           in Left $ CBOR.DecoderErrorCustom "Failed to deserialise key witness" allErrs
-        Right res -> return res
+  -- NB: built directly from 'Valid.Failure'/'Valid.Success' (rather than via
+  -- 'Valid.liftError'/'Valid.toEither') since those convenience functions were
+  -- removed from the "validation" package's newer, lens-based API; the
+  -- constructors and the 'Semigroup'/'Monoid' instances used here are stable
+  -- across both APIs.
+  case
+    mconcat $
+      map
+        (either (Valid.Failure . return) Valid.Success)
+        [ bootstrapWitnessDecoder bs
+        , shelleyKeyWitnessDecoder bs
+        , legacyKeyWitnessDecoder bs
+        ]
+    of
+    Valid.Failure errs ->
+      let allErrs = Text.unlines $ map renderBuildable errs
+       in Left $ CBOR.DecoderErrorCustom "Failed to deserialise key witness" allErrs
+    Valid.Success res -> return res
  where
   shelleyKeyWitnessDecoder b =
     ShelleyKeyWitness sbe