SRE Node 11.2 preparation

cardano-node-chairman/.changes/20261015_171029_palas_chairman_reads_cardano_config.yml

@@ -0,0 +1,4 @@
+pr: 6732
+kind:
+  - feature
+description: The chairman reads node configurations in the cardano-config format too, the way the node does.

cardano-node-chairman/app/Cardano/Chairman/Commands/Run.hs

@@ -1,3 +1,4 @@
+{-# LANGUAGE LambdaCase #-}
 {-# LANGUAGE NamedFieldPuns #-}
 {-# LANGUAGE OverloadedStrings #-}
 {-# LANGUAGE RankNTypes #-}
@@ -12,9 +13,14 @@ import           Cardano.Api
 import qualified Cardano.Api as Api
 
 import           Cardano.Chairman (chairmanTest)
+import           Cardano.Configuration (defaultCliArgs)
 import           Cardano.Ledger.BaseTypes (unNonZero)
+import           Cardano.Node.Configuration.CardanoConfigResolve (ConfigurationDialect (..),
+                   ResolvedNodeConfiguration (..), buildFromCardanoConfigWith,
+                   classifyConfigurationFile)
 import           Cardano.Node.Configuration.POM (PartialNodeConfiguration (..),
                    parseNodeConfigurationFP)
+import qualified Cardano.Node.Configuration.POM as POM
 import           Cardano.Node.Protocol
 import           Cardano.Node.Types
 import           Cardano.Prelude (ConvertText (..))
@@ -97,13 +103,7 @@ run RunOpts
     , caConfigYaml
     } = do
 
-  configYamlPc <- liftIO . parseNodeConfigurationFP $ Just caConfigYaml
-
-  ptclConfig <- case getProtocolConfiguration configYamlPc of
-                  Nothing ->
-                    error $ "Node protocol configuration was not specified "<>
-                            "in Config yaml filepath: " <> unConfigPath caConfigYaml
-                  Just ptclConfig -> return ptclConfig
+  ptclConfig <- readProtocolConfiguration caConfigYaml
 
   eitherSomeProtocol <- runExceptT $ mkConsensusProtocol ptclConfig Nothing
 
@@ -140,6 +140,26 @@ run RunOpts
       NodeProtocolConfigurationCardano{} ->
         CardanoModeParams $ EpochSlots $ unNonZero k
 
+-- | The protocol configuration in a node configuration file, read the way the
+-- node reads it: by @cardano-config@ for a file in its format, by the node's own
+-- parser otherwise. The chairman's command line is not the node's, so the
+-- settings all come from the file.
+readProtocolConfiguration :: ConfigYamlFilePath -> IO NodeProtocolConfiguration
+readProtocolConfiguration caConfigYaml =
+  classifyConfigurationFile configFile >>= \case
+    LegacyDialect -> do
+      configYamlPc <- parseNodeConfigurationFP $ Just caConfigYaml
+      case getProtocolConfiguration configYamlPc of
+        Nothing ->
+          error $ "Node protocol configuration was not specified "<>
+                  "in Config yaml filepath: " <> configFile
+        Just ptclConfig -> return ptclConfig
+    CardanoConfigDialect ->
+      POM.ncProtocolConfig . rncConfiguration
+        <$> buildFromCardanoConfigWith configFile (defaultCliArgs configFile)
+ where
+  configFile = unConfigPath caConfigYaml
+
   getProtocolConfiguration
     :: PartialNodeConfiguration
     -> Maybe NodeProtocolConfiguration

cardano-node-chairman/cardano-node-chairman.cabal

@@ -42,6 +42,7 @@ executable cardano-node-chairman
                         -rtsopts
                         "-with-rtsopts=-T"
   build-depends:        cardano-api
+                      , cardano-config ^>= 2.2
                       , cardano-crypto-class
                       , cardano-git-rev ^>= 0.2.2
                       , cardano-ledger-core

cardano-node/src/Cardano/Node/Configuration/CardanoConfigResolve.hs

@@ -31,25 +31,27 @@ module Cardano.Node.Configuration.CardanoConfigResolve
     ResolvedNodeConfiguration (..)
   , CrossCheck (..)
   , buildNodeConfiguration
+  , buildFromCardanoConfigWith
   , NodeConfigurationError (..)
 
     -- * Dialects
   , ConfigurationDialect (..)
   , classifyConfigurationFile
+  , tracingConfigSource
 
     -- * Command-line layers
   , nodeCliPartialConfiguration
   , nodeCliLayer
   , cardanoConfigCliLayer
   ) where
 
-import           Cardano.Logging.Types (TraceConfig)
+import           Cardano.Logging.ConfigurationParser (ConfigSource (..))
 import qualified Cardano.Configuration as Cfg
 import qualified Cardano.Configuration.CliArgs as CliArgs
-import           Cardano.Node.Configuration.CardanoConfigAdapter
-                   (cardanoConfigToNodeConfiguration)
-import           Cardano.Node.Configuration.CardanoConfigCompare
-                   (compareConfigurations, deprecatedFlagWarnings)
+import           Cardano.Logging.Types (TraceConfig)
+import           Cardano.Node.Configuration.CardanoConfigAdapter (cardanoConfigToNodeConfiguration)
+import           Cardano.Node.Configuration.CardanoConfigCompare (compareConfigurations,
+                   deprecatedFlagWarnings)
 import           Cardano.Node.Configuration.POM (NodeConfiguration (..),
                    PartialNodeConfiguration (..), defaultPartialNodeConfiguration,
                    makeNodeConfiguration, parseNodeConfigurationFP)
@@ -144,6 +146,22 @@ classifyConfigurationFile fp =
     Right (Object o) | KeyMap.member "Configuration" o -> pure CardanoConfigDialect
     _ -> pure LegacyDialect
 
+-- | How to hand a node configuration file's tracing settings to
+-- trace-dispatcher, for a tool with a file and no node to resolve it.
+--
+-- Reading a file, the library expects the tracing settings at its root. An
+-- envelope puts them at @Configuration.HermodTracing@, so hand over that object
+-- instead, which is what 'FromJSONObject' is for. Anything else goes as the
+-- file, as before.
+tracingConfigSource :: FilePath -> IO ConfigSource
+tracingConfigSource fp =
+  Yaml.decodeFileEither fp >>= \case
+    Right (Object top)
+      | Just (Object cfg) <- KeyMap.lookup "Configuration" top
+      , Just (Object tracing) <- KeyMap.lookup "HermodTracing" cfg ->
+          pure (FromJSONObject tracing)
+    _ -> pure (FromFile fp)
+
 -- | Resolve a legacy configuration with the node's own POM parser — the result
 -- the node runs on — and cross-check it against @cardano-config@'s.
 buildFromLegacy ::
@@ -167,10 +185,16 @@ buildFromLegacy crossCheck partialConf configFp = do
 -- the node's 'NodeConfiguration'. Unlike the cross-check on the legacy path,
 -- every failure here is fatal: there is no second parser to fall back to.
 buildFromCardanoConfig :: FilePath -> IO ResolvedNodeConfiguration
-buildFromCardanoConfig configFp = do
-  cliArgs <-
-    cardanoConfigCliArgs configFp
-      >>= either (Exception.throwIO . NodeConfigurationError) pure
+buildFromCardanoConfig configFp =
+  cardanoConfigCliArgs configFp
+    >>= either (Exception.throwIO . NodeConfigurationError) (buildFromCardanoConfigWith configFp)
+
+-- | 'buildFromCardanoConfig' with a given command-line layer instead of this
+-- process's own command line: for a caller that is not the node, whose command
+-- line is not the node's. With @'Cfg.defaultCliArgs' configFp@ every setting
+-- comes from the file.
+buildFromCardanoConfigWith :: FilePath -> Cfg.CliArgs -> IO ResolvedNodeConfiguration
+buildFromCardanoConfigWith configFp cliArgs = do
   (fileCfg, fileWarnings) <- Cfg.parseConfigurationFiles configFp
   (cfgNc, checkWarnings) <-
     either (Exception.throwIO . NodeConfigurationError . displayException) pure $

cardano-node/src/Cardano/Node/Tracing/Consistency.hs

@@ -27,6 +27,7 @@ import qualified Cardano.Network.PeerSelection.Governor.Types as Cardano
 import           Cardano.Network.PeerSelection.PeerTrustable (PeerTrustable)
 import           Cardano.Network.Tracing.PeerSelection ()
 import           Cardano.Network.Tracing.PeerSelectionCounters ()
+import           Cardano.Node.Configuration.CardanoConfigResolve (tracingConfigSource)
 import           Cardano.Node.Handlers.Shutdown (ShutdownTrace)
 import           Cardano.Node.Startup
 import           Cardano.Node.Tracing.DefaultTraceConfig (defaultCardanoConfig)
@@ -133,8 +134,9 @@ checkNodeTraceConfigurationWith ::
   -> FilePath
   -> IO NSWarnings
 checkNodeTraceConfigurationWith dt configFileName = do
+  src <- tracingConfigSource configFileName
   w1 <- checkTraceConfiguration
-          (FromFile configFileName)
+          src
           defaultCardanoConfig
           getAllNamespaces
   pure $ w1 <> dtWarnings dt

cardano-node/src/Cardano/Node/Tracing/Documentation.hs

@@ -35,6 +35,7 @@ import qualified Cardano.Network.PeerSelection.Governor.Types as Cardano
 import           Cardano.Network.PeerSelection.PeerTrustable (PeerTrustable (..))
 import           Cardano.Network.Tracing.PeerSelection ()
 import           Cardano.Network.Tracing.PeerSelectionCounters ()
+import           Cardano.Node.Configuration.CardanoConfigResolve (tracingConfigSource)
 import           Cardano.Node.Configuration.TopologyP2P ()
 import           Cardano.Node.Handlers.Shutdown (ShutdownTrace)
 import           Cardano.Node.Startup
@@ -189,6 +190,14 @@ runTraceDocumentationCmd
 runTraceDocumentationCmd TraceDocumentationCmd{..} = do
   docTracers tdcConfigFile tdcOutput tdMetricsHelp tdNamespaceList
 
+-- | Read the tracing configuration from a node configuration file. This command
+-- has no node, so there is no resolved configuration to use as there is at
+-- startup; 'tracingConfigSource' says how to present the file.
+resolveTracingConfig :: FilePath -> IO TraceConfig
+resolveTracingConfig fp = do
+    src <- tracingConfigSource fp
+    readConfigurationWithDefault src defaultCardanoConfig
+
 -- Have to repeat the construction of the tracers here,
 -- as the tracers are behind old tracer interface after construction in mkDispatchTracers.
 -- Can be changed, when old tracers have gone
@@ -215,7 +224,7 @@ docTracersFirstPhase :: forall blk peer remotePeer.
   -> IO (DocTracer, TraceConfig)
 docTracersFirstPhase condConfigFileName = do
     trConfig      <- case condConfigFileName of
-                        Just fn -> readConfigurationWithDefault (FromFile fn) defaultCardanoConfig
+                        Just fn -> resolveTracingConfig fn
                         Nothing -> pure defaultCardanoConfig
     let trBase    :: Logging.Trace IO FormattedMessage = docTracer (Stdout MachineFormat)
         trForward :: Logging.Trace IO FormattedMessage = docTracer Forwarder

cardano-submit-api/.changes/20261015_033428_palas_bump_cardano_api_11_9.yml

@@ -0,0 +1,4 @@
+pr: 6732
+kind:
+  - maintenance
+description: Bumped cardano-api to 11.11 and cardano-cli to 11.6.

cardano-testnet/.changes/20261015_033428_palas_bump_cardano_api_11_9.yml

@@ -0,0 +1,4 @@
+pr: 6732
+kind:
+  - maintenance
+description: Bumped cardano-api to 11.11 and cardano-cli to 11.6.

cardano-testnet/.changes/20261015_033946_palas_config_through_cardano_config.yml

@@ -0,0 +1,4 @@
+pr: 6732
+kind:
+  - breaking
+description: Testnet node configurations are written in the cardano-config format and through cardano-config, with their genesis hashes refreshed whenever the testnet starts and the node's default tracing options written out; removed createConfigJson, createConfigJsonNoHash, getByronGenesisHash, getShelleyGenesisHash, GenesisHashesPolicy and the configPath field of SubmitApiConf, and getStartTime no longer takes a path.

cardano-testnet/cardano-testnet.cabal

@@ -42,9 +42,8 @@ library
                       , ansi-terminal
                       , bytestring
                       , cardano-api ^>= 11.11
+                      , cardano-config ^>= 2.2.3
                       , cardano-cli:{cardano-cli, cardano-cli-test-lib} ^>= 11.6
-                      , cardano-crypto-class ^>=2.6
-                      , cardano-crypto-wrapper
                       , cardano-git-rev ^>= 0.2.2
                       , cardano-ledger-alonzo
                       , cardano-ledger-binary

cardano-testnet/src/Parsers/Run.hs

@@ -88,11 +88,7 @@ createEnvOptions CardanoTestnetCreateEnvOptions
   , createEnvOutputDir=outputDir
   } = do
       conf <- mkConfigAbs outputDir
-      void $ createTestnetEnv
-        creationOptions
-        -- Do not add hashes to the main config file, so that genesis files
-        -- can be modified without having to recompute hashes every time.
-        conf{genesisHashesPolicy = WithoutHashes}
+      void $ createTestnetEnv creationOptions conf
 
 runCardanoOptions :: CardanoTestnetCliOptions -> IO ()
 runCardanoOptions = \case

cardano-testnet/src/Testnet/Components/Configuration.hs

@@ -6,13 +6,11 @@
 {-# LANGUAGE TypeApplications #-}
 
 module Testnet.Components.Configuration
-  ( createConfigJson
-  , createConfigJsonNoHash
+  ( writeConfiguration
+  , refreshConfigurationHashes
   , createSPOGenesisAndFiles
   , numSeededUTxOKeys
 
-  , getByronGenesisHash
-  , getShelleyGenesisHash
   , getDefaultAlonzoGenesis
   , getDefaultShelleyGenesis
   , startTimeOffsetSeconds
@@ -24,24 +22,17 @@ module Testnet.Components.Configuration
 import           Cardano.Api hiding (Value, cardanoEra)
 import           Cardano.Api.Ledger (AlonzoGenesis, ConwayGenesis)
 
-import           Cardano.Chain.Genesis (GenesisHash (unGenesisHash), readGenesisData)
-import qualified Cardano.Crypto.Hash.Blake2b as Crypto
-import qualified Cardano.Crypto.Hash.Class as Crypto
+import           Cardano.Configuration.Edit (configurationDocumentFromValue,
+                   readConfigurationDocument, refreshGenesisHashes, writeConfigurationDocument)
 import           Cardano.Ledger.BaseTypes (unsafeNonZero)
 import           Cardano.Ledger.Dijkstra.Genesis (DijkstraGenesis)
-import           Cardano.Node.Protocol.Byron
 
 import           Control.Exception
 import           Control.Monad
 import           Control.Monad.Extra
 import           Data.Aeson
-import qualified Data.Aeson as Aeson
 import qualified Data.Aeson.Encode.Pretty as A
-import           Data.Aeson.Key hiding (fromString)
-import           Data.Aeson.KeyMap hiding (map)
-import qualified Data.ByteString as BS
 import qualified Data.ByteString.Lazy as LBS
-import qualified Data.Text as Text
 import qualified Data.Time.Clock as DTC
 import           Data.Word (Word64)
 import           GHC.Stack (HasCallStack)
@@ -61,62 +52,30 @@ import qualified Hedgehog.Extras.Stock.Time as DTC
 
 import           RIO (MonadThrow, throwM)
 
--- | Returns JSON encoded hashes of the era, as well as the hard fork configuration toggle.
-createConfigJson :: ()
+-- | Write the node configuration of a testnet: the default configuration for the
+-- given era, with the hashes of the genesis files in the same directory as the
+-- configuration file. The file is written in the @cardano-config@ format.
+writeConfiguration :: ()
   => HasCallStack
   => MonadIO m
-  => MonadThrow m
-  => TmpAbsolutePath
+  => FilePath -- ^ The configuration file to write
   -> ShelleyBasedEra era -- ^ The era used for generating the hard fork configuration toggle
-  -> m (KeyMap Aeson.Value)
-createConfigJson (TmpAbsolutePath tempAbsPath) sbe = GHC.withFrozenCallStack $ do
-  byronGenesisHash <- getByronGenesisHash $ tempAbsPath </> "byron-genesis.json"
-  shelleyGenesisHash <- getHash ShelleyEra "ShelleyGenesisHash"
-  alonzoGenesisHash  <- getHash AlonzoEra  "AlonzoGenesisHash"
-  conwayGenesisHash  <- getHash ConwayEra  "ConwayGenesisHash"
-  dijkstraGenesisHash  <- getHash DijkstraEra  "DijkstraGenesisHash"
-
-  pure $ mconcat
-    [ byronGenesisHash
-    , shelleyGenesisHash
-    , alonzoGenesisHash
-    , conwayGenesisHash
-    , dijkstraGenesisHash
-    , Defaults.defaultYamlHardforkViaConfig sbe
-    ]
-   where
-    getHash ::  MonadIO m => CardanoEra a -> Text.Text -> m (KeyMap Value)
-    getHash e = getShelleyGenesisHash (tempAbsPath </> Defaults.defaultGenesisFilepath e)
-
-createConfigJsonNoHash :: ()
-  => ShelleyBasedEra era -- ^ The era used for generating the hard fork configuration toggle
-  -> KeyMap Aeson.Value
-createConfigJsonNoHash = Defaults.defaultYamlHardforkViaConfig
-
--- Generate hashes for genesis.json files
-
-getByronGenesisHash
-  :: MonadIO m
-  => MonadThrow m
-  => FilePath
-  -> m (KeyMap Aeson.Value)
-getByronGenesisHash path = do
-  e <- runExceptT $ readGenesisData path
-  case e of
-    Left err -> throwM $ GenesisReadError path err
-    Right (_, genesisHash) -> do
-      let genesisHash' = unGenesisHash genesisHash
-      pure . singleton "ByronGenesisHash" $ toJSON genesisHash'
-
-getShelleyGenesisHash
-  :: MonadIO m
-  => FilePath
-  -> Text
-  -> m (KeyMap Aeson.Value)
-getShelleyGenesisHash path key = do
-  content <- liftIOAnnotated $ BS.readFile path
-  let genesisHash = Crypto.hashWith id content :: Crypto.Hash Crypto.Blake2b_256 BS.ByteString
-  pure . singleton (fromText key) $ toJSON genesisHash
+  -> m ()
+writeConfiguration configFile sbe = GHC.withFrozenCallStack . liftIOAnnotated $ do
+  (document, _warnings) <- configurationDocumentFromValue . Object $ Defaults.defaultYamlHardforkViaConfig sbe
+  refreshGenesisHashes (takeDirectory configFile) document >>= writeConfigurationDocument configFile
+
+-- | Set the genesis hashes in a node configuration file to the hashes of the
+-- genesis files it names, adding the missing ones and replacing the ones that
+-- are out of date.
+refreshConfigurationHashes :: ()
+  => HasCallStack
+  => MonadIO m
+  => FilePath -- ^ The configuration file
+  -> m ()
+refreshConfigurationHashes configFile = GHC.withFrozenCallStack . liftIOAnnotated $ do
+  (document, _warnings) <- readConfigurationDocument configFile
+  refreshGenesisHashes (takeDirectory configFile) document >>= writeConfigurationDocument configFile
 
 -- | For an unknown reason, CLI commands are a lot slower on Windows than on Linux and
 -- MacOS.  We need to allow a lot more time to set up a testnet.