Add CompatMode configuration (informalsystems#3667)

* Update test bootstrap to work with Celestia chain

* Update Nix flake and add Celestia CI job

* Add changelog entry

* Add guide section for 'compat_mode' configuration

* Apply suggestions from code review

Co-authored-by: Romain Ruetschi <[email protected]>
Signed-off-by: Luca Joss <[email protected]>

* Implement serialization and deserialization for CompatMode

* Update crates/relayer/src/util/compat_mode.rs

Co-authored-by: Romain Ruetschi <[email protected]>
Signed-off-by: Luca Joss <[email protected]>

---------

Signed-off-by: Luca Joss <[email protected]>
Co-authored-by: Romain Ruetschi <[email protected]>

.changelog/unreleased/features/ibc-relayer/3623-compat-mode-configurability.md

@@ -0,0 +1,3 @@
+- Add an optional per-chain setting `compat_mode`, which can be
+  used to specify which CometBFT compatibility mode is used for interacting with the node over RPC.
+  ([\#3623](https://github.com/informalsystems/hermes/issues/3623))

.github/workflows/integration.yaml

@@ -547,6 +547,51 @@ jobs:
             cargo nextest run -p ibc-integration-test --no-fail-fast --failure-output final --test-threads=2 \
             --features interchain-security,ics31 interchain_security::
 
+  celestia-to-gaia:
+    runs-on: ubuntu-20.04
+    strategy:
+      fail-fast: false
+      matrix:
+        chain:
+          - package: celestia
+            command: celestia-appd
+            account_prefix: celestia
+            native_token: utia
+    steps:
+      - uses: actions/checkout@v4
+      - uses: cachix/install-nix-action@v22
+        with:
+          install_url: https://nixos-nix-install-tests.cachix.org/serve/vij683ly7sl95nnhb67bdjjfabclr85m/install
+          install_options: '--tarball-url-prefix https://nixos-nix-install-tests.cachix.org/serve'
+          extra_nix_config: |
+            experimental-features = nix-command flakes
+      - uses: cachix/cachix-action@v12
+        with:
+          name: cosmos
+      - uses: actions-rs/toolchain@v1
+        with:
+          toolchain: stable
+          override: true
+      - uses: Swatinem/rust-cache@v2
+      - uses: actions-rs/cargo@v1
+        with:
+          command: test
+          args: -p ibc-integration-test --features interchain-security --no-fail-fast --no-run
+      - name: Install cargo-nextest
+        run: curl -LsSf https://get.nexte.st/latest/linux | tar zxf - -C ${CARGO_HOME:-~/.cargo}/bin
+      - env:
+          RUST_LOG: info
+          RUST_BACKTRACE: 1
+          NO_COLOR_LOG: 1
+          COMPAT_MODES: 0.34
+          CHAIN_COMMAND_PATHS: ${{ matrix.chain.command }},gaiad
+          ACCOUNT_PREFIXES: ${{ matrix.chain.account_prefix }},cosmos
+          NATIVE_TOKENS: ${{ matrix.chain.native_token }},stake
+        run: |
+          nix shell .#python .#gaia12 .#${{ matrix.chain.package }} -c \
+            cargo nextest run -p ibc-integration-test --no-fail-fast --failure-output final --test-threads=2 \
+            --features celestia
+
   model-based-test:
     runs-on: ubuntu-20.04
     timeout-minutes: 60

config.toml

@@ -377,6 +377,15 @@ memo_prefix = ''
 # submitted to this chain.
 # fee_granter = ''
 
+# Specify the CometBFT compatibility mode to use.
+# The following behaviours are applied whether the `compat_mode` is configured or not:
+#   * compat_mode is specified and the version queried from /status is the same as the one configured: Use that version without log output
+#   * compat_mode is specified but the version queried from /status differs: The CompatMode configured is used, but a warning log is outputted
+#   * compat_mode is not specified but /status returns a correct version: The CompatMode retrieved from the endpoint is used
+#   * compat_mode is not specified and /status does not return a valid version: Hermes stops and outputs an error informing the user a compat_mode needs to be configured
+# Possible values: [`0.34`, `0.37`]
+# compat_mode = '0.34'
+
 # Specify the a clear interval for the chain.
 # This will override the global clear interval for this chain only, allowing different intervals for each chain.
 # clear_interval = 50

crates/relayer-cli/src/chain_registry.rs

@@ -160,6 +160,7 @@ where
         address_type: AddressType::default(),
         sequential_batch_tx: false,
         extension_options: Vec::new(),
+        compat_mode: None,
         clear_interval: None,
     })
 }

crates/relayer-cli/src/commands/listen.rs

@@ -17,6 +17,7 @@ use ibc_relayer::{
     chain::handle::Subscription,
     config::{ChainConfig, EventSourceMode},
     event::source::EventSource,
+    util::compat_mode::compat_mode_from_version,
 };
 use ibc_relayer_types::{core::ics24_host::identifier::ChainId, events::IbcEvent};
 
@@ -171,10 +172,8 @@ fn detect_compatibility_mode(
 ) -> eyre::Result<CompatMode> {
     let client = HttpClient::new(config.rpc_addr.clone())?;
     let status = rt.block_on(client.status())?;
-    let compat_mode = CompatMode::from_version(status.node_info.version).unwrap_or_else(|e| {
-        warn!("Unsupported tendermint version, will use v0.34 compatibility mode but relaying might not work as desired: {e}");
-        CompatMode::V0_34
-    });
+    let compat_mode =
+        compat_mode_from_version(&config.compat_mode, status.node_info.version)?.into();
     Ok(compat_mode)
 }
 

crates/relayer/src/chain/cosmos.rs

@@ -100,6 +100,7 @@ use crate::keyring::{KeyRing, Secp256k1KeyPair, SigningKeyPair};
 use crate::light_client::tendermint::LightClient as TmLightClient;
 use crate::light_client::{LightClient, Verified};
 use crate::misbehaviour::MisbehaviourEvidence;
+use crate::util::compat_mode::compat_mode_from_version;
 use crate::util::pretty::{
     PrettyIdentifiedChannel, PrettyIdentifiedClientState, PrettyIdentifiedConnection,
 };
@@ -875,10 +876,7 @@ impl ChainEndpoint for CosmosSdkChain {
 
         let node_info = rt.block_on(fetch_node_info(&rpc_client, &config))?;
 
-        let compat_mode = CompatMode::from_version(node_info.version).unwrap_or_else(|e| {
-            warn!("Unsupported tendermint version, will use v0.34 compatibility mode but relaying might not work as desired: {e}");
-            CompatMode::V0_34
-        });
+        let compat_mode = compat_mode_from_version(&config.compat_mode, node_info.version)?.into();
         rpc_client.set_compat_mode(compat_mode);
 
         let light_client = TmLightClient::from_config(&config, node_info.id)?;

crates/relayer/src/config.rs

@@ -1,5 +1,6 @@
 //! Relayer configuration
 
+pub mod compat_mode;
 pub mod error;
 pub mod filter;
 pub mod gas_multiplier;
@@ -32,6 +33,7 @@ use ibc_relayer_types::core::ics24_host::identifier::{ChainId, ChannelId, PortId
 use ibc_relayer_types::timestamp::ZERO_DURATION;
 
 use crate::chain::ChainType;
+use crate::config::compat_mode::CompatMode;
 use crate::config::gas_multiplier::GasMultiplier;
 use crate::config::types::{MaxMsgNum, MaxTxSize, Memo};
 use crate::error::Error as RelayerError;
@@ -663,6 +665,7 @@ pub struct ChainConfig {
     pub address_type: AddressType,
     #[serde(default = "Vec::new", skip_serializing_if = "Vec::is_empty")]
     pub extension_options: Vec<ExtensionOption>,
+    pub compat_mode: Option<CompatMode>,
     pub clear_interval: Option<u64>,
 }
 

crates/relayer/src/config/compat_mode.rs

@@ -0,0 +1,97 @@
+use core::fmt::{Display, Error as FmtError, Formatter};
+use core::str::FromStr;
+use serde::Deserialize;
+use serde::Deserializer;
+use serde::Serialize;
+use serde::Serializer;
+
+use tendermint_rpc::client::CompatMode as TmCompatMode;
+
+use crate::config::Error;
+
+/// CometBFT RPC compatibility mode
+///
+/// Can be removed in favor of the one in tendermint-rs, once
+/// <https://github.com/informalsystems/tendermint-rs/pull/1367> is merged.
+#[derive(Clone, Debug, Eq, PartialEq)]
+pub enum CompatMode {
+    /// Use version 0.34 of the protocol.
+    V0_34,
+    /// Use version 0.37 of the protocol.
+    V0_37,
+}
+
+impl Display for CompatMode {
+    fn fmt(&self, f: &mut Formatter<'_>) -> Result<(), FmtError> {
+        match self {
+            Self::V0_34 => write!(f, "v0.34"),
+            Self::V0_37 => write!(f, "v0.37"),
+        }
+    }
+}
+
+impl FromStr for CompatMode {
+    type Err = Error;
+
+    fn from_str(s: &str) -> Result<Self, Self::Err> {
+        const VALID_COMPAT_MODES: &str = "0.34, 0.37";
+
+        // Trim leading 'v', if present
+        match s.trim_start_matches('v') {
+            "0.34" => Ok(CompatMode::V0_34),
+            "0.37" => Ok(CompatMode::V0_37),
+            _ => Err(Error::invalid_compat_mode(
+                s.to_string(),
+                VALID_COMPAT_MODES,
+            )),
+        }
+    }
+}
+
+impl<'de> Deserialize<'de> for CompatMode {
+    fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
+    where
+        D: Deserializer<'de>,
+    {
+        use serde::de;
+
+        let s = String::deserialize(deserializer)?;
+        FromStr::from_str(&s).map_err(de::Error::custom)
+    }
+}
+
+impl Serialize for CompatMode {
+    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
+    where
+        S: Serializer,
+    {
+        self.to_string().serialize(serializer)
+    }
+}
+
+impl From<TmCompatMode> for CompatMode {
+    fn from(value: TmCompatMode) -> Self {
+        match value {
+            TmCompatMode::V0_34 => Self::V0_34,
+            TmCompatMode::V0_37 => Self::V0_37,
+        }
+    }
+}
+
+impl From<CompatMode> for TmCompatMode {
+    fn from(value: CompatMode) -> Self {
+        match value {
+            CompatMode::V0_34 => Self::V0_34,
+            CompatMode::V0_37 => Self::V0_37,
+        }
+    }
+}
+
+impl CompatMode {
+    pub fn equal_to_tm_compat_mode(&self, tm_compat_mode: TmCompatMode) -> bool {
+        match self {
+            Self::V0_34 => tm_compat_mode == TmCompatMode::V0_34,
+            Self::V0_37 => tm_compat_mode == TmCompatMode::V0_37,
+        }
+    }
+}

crates/relayer/src/config/error.rs

@@ -10,6 +10,10 @@ define_error! {
             [ TraceError<toml::de::Error> ]
             |_| { "invalid configuration" },
 
+        InvalidCompatMode
+            { compat_mode: String, valide_modes: &'static str }
+            |e| { format!("invalid compatibility mode: '{}' (supported: {})", e.compat_mode, e.valide_modes) },
+
         Encode
             [ TraceError<toml::ser::Error> ]
             |_| { "invalid configuration" },

crates/relayer/src/error.rs

@@ -582,6 +582,10 @@ define_error! {
             { address: String }
             [ TendermintRpcError ]
             |e| { format!("invalid archive node address {}", e.address) },
+
+        InvalidCompatMode
+            [ TendermintRpcError ]
+            |_| { "Invalid CompatMode queried from chain and no `compat_mode` configured in Hermes. This can be fixed by specifying a `compat_mode` in Hermes config.toml" },
     }
 }
 

crates/relayer/src/util.rs

@@ -2,6 +2,7 @@ mod block_on;
 pub use block_on::{block_on, spawn_blocking};
 
 pub mod collate;
+pub mod compat_mode;
 pub mod debug_section;
 pub mod diff;
 pub mod iter;

crates/relayer/src/util/compat_mode.rs

@@ -0,0 +1,27 @@
+use tracing::warn;
+
+use tendermint::Version;
+use tendermint_rpc::client::CompatMode as TmCompatMode;
+
+use crate::config::compat_mode::CompatMode;
+use crate::error::Error;
+
+/// This is a wrapper around tendermint-rs CompatMode::from_version() method.
+///
+pub fn compat_mode_from_version(
+    configured_version: &Option<CompatMode>,
+    version: Version,
+) -> Result<CompatMode, Error> {
+    let queried_version = TmCompatMode::from_version(version);
+
+    // This will prioritize the use of the CompatMode specified in Hermes configuration file
+    match (configured_version, queried_version) {
+        (Some(configured), Ok(queried)) if !configured.equal_to_tm_compat_mode(queried) => {
+            warn!("be wary of potential `compat_mode` misconfiguration. Configured version: {}, chain version: {}. Hermes will use the configured `compat_mode` version `{}`. If this configuration is done on purpose this message can be ignored.", configured, queried, configured);
+            Ok(configured.clone())
+        }
+        (Some(configured), _) => Ok(configured.clone()),
+        (_, Ok(queried)) => Ok(queried.into()),
+        (_, Err(e)) => Err(Error::invalid_compat_mode(e)),
+    }
+}

flake.nix

@@ -48,6 +48,7 @@
           stride-consumer
           migaloo
           neutron
+          celestia
           ;
 
         python = nixpkgs.python3.withPackages (p: [

guide/src/SUMMARY.md

@@ -33,6 +33,7 @@
     - [Filter incentivized packets](./documentation/configuration/filter-incentivized.md)
     - [Packet clearing](./documentation/configuration/packet-clearing.md)
     - [Performance tuning](./documentation/configuration/performance.md)
+    - [CometBFT Compatibility modes](./documentation/configuration/comet-compat-mode.md)
 
 - [Telemetry](./documentation/telemetry/index.md)
     - [Operators guide](./documentation/telemetry/operators.md)

guide/src/documentation/configuration/comet-compat-mode.md

@@ -0,0 +1,24 @@
+# CometBFT Compatibility modes
+
+## Overview
+
+There are two different compatibility modes for CometBFT, one for version v0.34 and one for versions v0.37 and v0.38. In order to verify the compatiblity used Hermes queries the node's `/status` endpoint, which contains the CometBFT version used. This can be an issue if a chain uses a custom version which does not output the version string Hermes expects. To still be able to relay for these chains a configuration can be set in Hermes.
+
+## Configuration
+
+The configuration is set per chain and can take two values `0.34` and `0.37`, other values will be invalid:
+
+```toml
+[[chains]]
+...
+compat_mode = '0.34'
+```
+
+Hermes will act in the following way whether or not the configuration is set:
+
+* `compat_mode` is specified and the version queried from `/status` is the same as the one configured: Use that version without log output
+* `compat_mode` is specified but the version queried from `/status` differs: The compatibility mode configured is used, but a warning log is outputted
+* `compat_mode` is not specified but /status returns a correct version: The compatibility mode  retrieved from the endpoint is used
+* `compat_mode` is not specified and /status does not return a valid version: Hermes stops and outputs an error informing the user that the `compat_mode` needs to be configured
+
+The configuration can also be found in the example [config.toml](https://github.com/informalsystems/hermes/blob/{{#include ../../templates/hermes-version.md}}/config.toml#382)

guide/src/documentation/configuration/index.md

@@ -19,4 +19,7 @@ This section includes everything you need to know to configure Hermes.
     * Description on packet clearing configurations
 
 - **[Performance Tuning](./performance.md)**
-    * Learn about configurations allowing more refined performance tuning.
+    * Learn about configurations allowing more refined performance tuning.
+
+- **[CometBFT Compatibility modes](./comet-compat-mode.md)**
+    * Handle different CometBFT compatibility modes.

tools/integration-test/Cargo.toml

@@ -39,6 +39,7 @@ ics31 = []
 clean-workers = []
 fee-grant = []
 interchain-security = []
+celestia = []
 
 [[bin]]
 name = "test_setup_with_binary_channel"