feat: improve datanode snapshot creation #11396
+507
−56
Draft
Jenkins MKII / System Tests Capsule
failed
Jul 9, 2024 in 0s
failed: 107, skipped: 2, passed: 4
Send us feedback
Details
tests.newLnl.lnl_test.test_restart_from_checkpoint_basic
failed on setup with "AssertionError: Network not running - unable to establish port offsets - check capsule"
Stack trace
@pytest.fixture(scope="session")
def channel_setup():
"""Setups up the connections for the test session."""
# Open the connection to vega.
logging.info(f"Attempting to establish connection with test network...")
> setup_base_urls()
conftest.py:420:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
settings.py:275: in setup_base_urls
BASE_URL = get_base_urls()
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
def get_base_urls():
ports_definition = {}
wallet_service_grpc_hosts = []
network_addr = "127.0.0.1"
ganache_addr = "127.0.0.1"
if len(VEGACAPSULE_NET_ADDR) > 0:
logging.info(f"INFO: Using VEGACAPSULE_NET_ADDR: {VEGACAPSULE_NET_ADDR}")
network_addr = VEGACAPSULE_NET_ADDR.strip()
# Patch for WSL2. This is required because ganache is working in different
# network than rest of the vega network because of docker.
ganache_addr = "127.0.0.1" if check_network("127.0.0.1", 8545) else network_addr
elif check_network("127.0.0.1"):
logging.info("INFO: Detected network running on address 127.0.0.1")
network_addr = "127.0.0.1"
ganache_addr = network_addr
if MAINNET_TEST_CASE:
# The mainnet port definition. To see more details read the documentation in the
# vegacapsule/readme.md file.
ports_definition = {
"data_grpc_node": 6314,
"data_gql_node": 6614,
"data_rest_url": 7314,
"core_grpc_node": 3000,
"core_rest_url": 3300,
"tendermint_node": 26000,
"walletserver": 1789,
"faucetserver": 1790,
}
elif NO_DATA_NODE_TEST_CASE:
ports_definition = {
"data_grpc_node": -1,
"data_gql_node": -1,
"data_rest_url": -1,
"core_grpc_node": 3002,
"core_rest_url": 3003,
"tendermint_node": 26657,
"walletserver": 1789,
"faucetserver": 1790,
}
else:
# Find the data-node. Its ports move depending on how many nodes are in the network.
# Ideally we should just ask vegacapsule what the data-nodes ports are but settings.py is
# imported everywhere as a bottom layer, and trying to call vegacapsule here just creates
# import cycles. It would take a big refactor to untangle, so it is not a task for now...
offset = None
for i in range(0, 10):
if check_network(network_addr, str(3008 + (i * 10))):
offset = int(i)
break
if threading.current_thread().name == "MainThread":
# Don't abort if we're on a child thread. Let whatever is destined to fail, fail later. This would indicate
# that we previously had a network, and it is now dead for some reason. Currently, this check is run at
# import time, which causes an infinite loop of dying worker threads if we error here.
> assert offset is not None, f"Network not running - unable to establish port offsets - check capsule"
E AssertionError: Network not running - unable to establish port offsets - check capsule
settings.py:204: AssertionError
-------------------- ERROR FOUND WITH NETWORK. ERROR 1 of 1 --------------------
Vegacapsule reports no network nodes found! Is the network completely dead?
---- Error detecting network errors during test fail! See logs for details. ----
'NoneType' object has no attribute 'get'
tests.newLnl.lnl_test.test_orders_and_positions_not_maintained_across_checkpoints
failed on setup with "AssertionError: Network not running - unable to establish port offsets - check capsule"
Stack trace
@pytest.fixture(scope="session")
def channel_setup():
"""Setups up the connections for the test session."""
# Open the connection to vega.
logging.info(f"Attempting to establish connection with test network...")
> setup_base_urls()
conftest.py:420:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
settings.py:275: in setup_base_urls
BASE_URL = get_base_urls()
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
def get_base_urls():
ports_definition = {}
wallet_service_grpc_hosts = []
network_addr = "127.0.0.1"
ganache_addr = "127.0.0.1"
if len(VEGACAPSULE_NET_ADDR) > 0:
logging.info(f"INFO: Using VEGACAPSULE_NET_ADDR: {VEGACAPSULE_NET_ADDR}")
network_addr = VEGACAPSULE_NET_ADDR.strip()
# Patch for WSL2. This is required because ganache is working in different
# network than rest of the vega network because of docker.
ganache_addr = "127.0.0.1" if check_network("127.0.0.1", 8545) else network_addr
elif check_network("127.0.0.1"):
logging.info("INFO: Detected network running on address 127.0.0.1")
network_addr = "127.0.0.1"
ganache_addr = network_addr
if MAINNET_TEST_CASE:
# The mainnet port definition. To see more details read the documentation in the
# vegacapsule/readme.md file.
ports_definition = {
"data_grpc_node": 6314,
"data_gql_node": 6614,
"data_rest_url": 7314,
"core_grpc_node": 3000,
"core_rest_url": 3300,
"tendermint_node": 26000,
"walletserver": 1789,
"faucetserver": 1790,
}
elif NO_DATA_NODE_TEST_CASE:
ports_definition = {
"data_grpc_node": -1,
"data_gql_node": -1,
"data_rest_url": -1,
"core_grpc_node": 3002,
"core_rest_url": 3003,
"tendermint_node": 26657,
"walletserver": 1789,
"faucetserver": 1790,
}
else:
# Find the data-node. Its ports move depending on how many nodes are in the network.
# Ideally we should just ask vegacapsule what the data-nodes ports are but settings.py is
# imported everywhere as a bottom layer, and trying to call vegacapsule here just creates
# import cycles. It would take a big refactor to untangle, so it is not a task for now...
offset = None
for i in range(0, 10):
if check_network(network_addr, str(3008 + (i * 10))):
offset = int(i)
break
if threading.current_thread().name == "MainThread":
# Don't abort if we're on a child thread. Let whatever is destined to fail, fail later. This would indicate
# that we previously had a network, and it is now dead for some reason. Currently, this check is run at
# import time, which causes an infinite loop of dying worker threads if we error here.
> assert offset is not None, f"Network not running - unable to establish port offsets - check capsule"
E AssertionError: Network not running - unable to establish port offsets - check capsule
settings.py:204: AssertionError
-------------------- ERROR FOUND WITH NETWORK. ERROR 1 of 1 --------------------
Vegacapsule reports no network nodes found! Is the network completely dead?
---- Error detecting network errors during test fail! See logs for details. ----
'NoneType' object has no attribute 'get'
tests.newLnl.lnl_test.test_withdrawal_and_deposit
failed on setup with "AssertionError: Network not running - unable to establish port offsets - check capsule"
Stack trace
@pytest.fixture(scope="session")
def channel_setup():
"""Setups up the connections for the test session."""
# Open the connection to vega.
logging.info(f"Attempting to establish connection with test network...")
> setup_base_urls()
conftest.py:420:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
settings.py:275: in setup_base_urls
BASE_URL = get_base_urls()
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
def get_base_urls():
ports_definition = {}
wallet_service_grpc_hosts = []
network_addr = "127.0.0.1"
ganache_addr = "127.0.0.1"
if len(VEGACAPSULE_NET_ADDR) > 0:
logging.info(f"INFO: Using VEGACAPSULE_NET_ADDR: {VEGACAPSULE_NET_ADDR}")
network_addr = VEGACAPSULE_NET_ADDR.strip()
# Patch for WSL2. This is required because ganache is working in different
# network than rest of the vega network because of docker.
ganache_addr = "127.0.0.1" if check_network("127.0.0.1", 8545) else network_addr
elif check_network("127.0.0.1"):
logging.info("INFO: Detected network running on address 127.0.0.1")
network_addr = "127.0.0.1"
ganache_addr = network_addr
if MAINNET_TEST_CASE:
# The mainnet port definition. To see more details read the documentation in the
# vegacapsule/readme.md file.
ports_definition = {
"data_grpc_node": 6314,
"data_gql_node": 6614,
"data_rest_url": 7314,
"core_grpc_node": 3000,
"core_rest_url": 3300,
"tendermint_node": 26000,
"walletserver": 1789,
"faucetserver": 1790,
}
elif NO_DATA_NODE_TEST_CASE:
ports_definition = {
"data_grpc_node": -1,
"data_gql_node": -1,
"data_rest_url": -1,
"core_grpc_node": 3002,
"core_rest_url": 3003,
"tendermint_node": 26657,
"walletserver": 1789,
"faucetserver": 1790,
}
else:
# Find the data-node. Its ports move depending on how many nodes are in the network.
# Ideally we should just ask vegacapsule what the data-nodes ports are but settings.py is
# imported everywhere as a bottom layer, and trying to call vegacapsule here just creates
# import cycles. It would take a big refactor to untangle, so it is not a task for now...
offset = None
for i in range(0, 10):
if check_network(network_addr, str(3008 + (i * 10))):
offset = int(i)
break
if threading.current_thread().name == "MainThread":
# Don't abort if we're on a child thread. Let whatever is destined to fail, fail later. This would indicate
# that we previously had a network, and it is now dead for some reason. Currently, this check is run at
# import time, which causes an infinite loop of dying worker threads if we error here.
> assert offset is not None, f"Network not running - unable to establish port offsets - check capsule"
E AssertionError: Network not running - unable to establish port offsets - check capsule
settings.py:204: AssertionError
-------------------- ERROR FOUND WITH NETWORK. ERROR 1 of 1 --------------------
Vegacapsule reports no network nodes found! Is the network completely dead?
---- Error detecting network errors during test fail! See logs for details. ----
'NoneType' object has no attribute 'get'
tests.newLnl.lnl_test.test_stake_added_or_removed_during_checkpoint_outage
failed on setup with "AssertionError: Network not running - unable to establish port offsets - check capsule"
Stack trace
@pytest.fixture(scope="session")
def channel_setup():
"""Setups up the connections for the test session."""
# Open the connection to vega.
logging.info(f"Attempting to establish connection with test network...")
> setup_base_urls()
conftest.py:420:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
settings.py:275: in setup_base_urls
BASE_URL = get_base_urls()
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
def get_base_urls():
ports_definition = {}
wallet_service_grpc_hosts = []
network_addr = "127.0.0.1"
ganache_addr = "127.0.0.1"
if len(VEGACAPSULE_NET_ADDR) > 0:
logging.info(f"INFO: Using VEGACAPSULE_NET_ADDR: {VEGACAPSULE_NET_ADDR}")
network_addr = VEGACAPSULE_NET_ADDR.strip()
# Patch for WSL2. This is required because ganache is working in different
# network than rest of the vega network because of docker.
ganache_addr = "127.0.0.1" if check_network("127.0.0.1", 8545) else network_addr
elif check_network("127.0.0.1"):
logging.info("INFO: Detected network running on address 127.0.0.1")
network_addr = "127.0.0.1"
ganache_addr = network_addr
if MAINNET_TEST_CASE:
# The mainnet port definition. To see more details read the documentation in the
# vegacapsule/readme.md file.
ports_definition = {
"data_grpc_node": 6314,
"data_gql_node": 6614,
"data_rest_url": 7314,
"core_grpc_node": 3000,
"core_rest_url": 3300,
"tendermint_node": 26000,
"walletserver": 1789,
"faucetserver": 1790,
}
elif NO_DATA_NODE_TEST_CASE:
ports_definition = {
"data_grpc_node": -1,
"data_gql_node": -1,
"data_rest_url": -1,
"core_grpc_node": 3002,
"core_rest_url": 3003,
"tendermint_node": 26657,
"walletserver": 1789,
"faucetserver": 1790,
}
else:
# Find the data-node. Its ports move depending on how many nodes are in the network.
# Ideally we should just ask vegacapsule what the data-nodes ports are but settings.py is
# imported everywhere as a bottom layer, and trying to call vegacapsule here just creates
# import cycles. It would take a big refactor to untangle, so it is not a task for now...
offset = None
for i in range(0, 10):
if check_network(network_addr, str(3008 + (i * 10))):
offset = int(i)
break
if threading.current_thread().name == "MainThread":
# Don't abort if we're on a child thread. Let whatever is destined to fail, fail later. This would indicate
# that we previously had a network, and it is now dead for some reason. Currently, this check is run at
# import time, which causes an infinite loop of dying worker threads if we error here.
> assert offset is not None, f"Network not running - unable to establish port offsets - check capsule"
E AssertionError: Network not running - unable to establish port offsets - check capsule
settings.py:204: AssertionError
-------------------- ERROR FOUND WITH NETWORK. ERROR 1 of 1 --------------------
Vegacapsule reports no network nodes found! Is the network completely dead?
---- Error detecting network errors during test fail! See logs for details. ----
'NoneType' object has no attribute 'get'
tests.API.grpc_streams_test.test_margin_levels_stream[cross_margin-None]
failed on setup with "AssertionError: Failed to start all scanners. Please check the logs.
assert not True"
Stack trace
request = <SubRequest 'set_up' for <Function test_margin_levels_stream[cross_margin-None]>>
channel_setup = None
context = {'app_state.network_parameters': {'auction.LongBlock': '{"threshold_and_duration":[{"threshold":"24h","duration":"6h"}...41b835d9bfa3a8535a9c221bafc09ab3fe', '513D65AC08': '963228ba16583b826508c6d94c67b9e0abfaab838fdae64a601e33a6d6b81209'}}
@pytest.fixture(scope="session", autouse=True)
def set_up(request, channel_setup, context):
st = f"Setup Session {request.node.name}"
logging.info("*" * len(st))
logging.info(st)
logging.info("*" * len(st))
logging.info(f"using default product: {DEFAULT_PRODUCT}")
session_start_time = time.time()
logging.info(f"Scanner cleanup")
# assert os.path.exists(VEGATOOLS), "Please ensure vegatools is available on the path."
# logging.info(f"Using vegatools found at: {VEGATOOLS}")
kill_vegatools()
clean_eventscanner_dir() # Can only do this once existing processes have been killed
if NO_DATA_NODE_TEST_CASE:
yield
return
> start_scanners()
conftest.py:473:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
conftest.py:270: in start_scanners
check_scanner_for_errors(event_scanner)
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
popen = <Popen: returncode: 1 args: ['/jenkins/workspace/common/system-tests-wrapper...>
def check_scanner_for_errors(popen):
timeout = 10
now = time.time()
# Scanner opens logs for each file type on the fly. If we can find the being block file, that means we're listening
# and will create each subsequent file as required.
eventscanner_path = f"{ROOT_DIR}/eventscanner/"
begin_block_log_file = pathlib.Path(eventscanner_path, "BUS_EVENT_TYPE_BEGIN_BLOCK.txt")
ps = None
logging.info(f"Waiting for begin_block log file to show scanner is running... Waiting up to {timeout} seconds.")
found_log = False
while time.time() <= now + timeout:
ps = get_vega_stream_processes()
if len(ps) < 1:
logging.info(f"Waiting for {1} vega stream processes. Found: {len(ps)}")
time.sleep(0.1)
continue
if begin_block_log_file.exists():
logging.info("Found log file. Scanner is running.")
found_log = True
break
else:
logging.info(f"No log file found at: {begin_block_log_file}.")
# TODO hack, if nullchain make some blocks happen
if NULL_BLOCK_CHAIN:
nullchain.move_forward(10)
time.sleep(0.1)
continue
logging.info(
f"Finished looking for logs and vegatools stream process. Found: {found_log}. Now checking error states of scanner process."
)
# Save this result, but first check all the popens, since their stdout/err gives more useful logging.
failed = False
if popen.poll() is not None:
failed = True
out, err = popen.communicate()
logging.error(f"Error starting scanner: \n{out}\n{err}")
> assert not failed, "Failed to start all scanners. Please check the logs."
E AssertionError: Failed to start all scanners. Please check the logs.
E assert not True
conftest.py:342: AssertionError
-------------------- ERROR FOUND WITH NETWORK. ERROR 1 of 1 --------------------
Trace from file: /jenkins/workspace/common/system-tests-wrapper/networkdata/testnet/logs/testnet-nodeset-full-2-full/data-node-2.stderr-2024-07-09T10:27:10Z.log
panic: failed to snapshot data
goroutine 728 [running]:
go.uber.org/zap/zapcore.CheckWriteAction.OnWrite(0x1?, 0x1?, {0x0?, 0x0?, 0xc00425adc0?})
/jenkins/GOPATH/pkg/mod/go.uber.org/[email protected]/zapcore/entry.go:196 +0x54
go.uber.org/zap/zapcore.(*CheckedEntry).Write(0xc000d08270, {0xc005b3a080, 0x1, 0x1})
/jenkins/GOPATH/pkg/mod/go.uber.org/[email protected]/zapcore/entry.go:262 +0x3ec
go.uber.org/zap.(*Logger).Panic(0xc00436aed0?, {0x5e25a34?, 0xe?}, {0xc005b3a080, 0x1, 0x1})
/jenkins/GOPATH/pkg/mod/go.uber.org/[email protected]/logger.go:258 +0x51
code.vegaprotocol.io/vega/datanode/networkhistory/snapshot.(*Service).createNewSnapshot.func3()
/jenkins/workspace/common/system-tests-wrapper/vega/datanode/networkhistory/snapshot/service_create_snapshot.go:139 +0x29f
code.vegaprotocol.io/vega/datanode/networkhistory/snapshot.(*Service).createNewSnapshot(0xc00119c000, {0x66596e0?, 0xc000aac0f0}, {0xc00112fe30, 0xb}, 0xc00387bf20?, 0x1)
/jenkins/workspace/common/system-tests-wrapper/vega/datanode/networkhistory/snapshot/service_create_snapshot.go:150 +0x7ff
code.vegaprotocol.io/vega/datanode/networkhistory/snapshot.(*Service).CreateSnapshotAsynchronously(...)
/jenkins/workspace/common/system-tests-wrapper/vega/datanode/networkhistory/snapshot/service_create_snapshot.go:54
code.vegaprotocol.io/vega/datanode/networkhistory/snapshot.(*Service).SnapshotData(0x0?, {0x66596e0?, 0xc000aac0f0?}, {0xc00112fe30?, 0x0?}, 0xc0011c5db0?)
/jenkins/workspace/common/system-tests-wrapper/vega/datanode/networkhistory/snapshot/service.go:94 +0x3b
code.vegaprotocol.io/vega/datanode/networkhistory.(*BlockCommitHandler).OnBlockCommitted.func1()
/jenkins/workspace/common/system-tests-wrapper/vega/datanode/networkhistory/block_commit_handler.go:66 +0x3c
github.com/cenkalti/backoff.RetryNotify(0xc003503d98, {0x6640948, 0xc001479a40}, 0x0)
/jenkins/GOPATH/pkg/mod/github.com/cenkalti/[email protected]+incompatible/retry.go:37 +0x164
github.com/cenkalti/backoff.Retry(...)
/jenkins/GOPATH/pkg/mod/github.com/cenkalti/[email protected]+incompatible/retry.go:24
code.vegaprotocol.io/vega/datanode/networkhistory.(*BlockCommitHandler).OnBlockCommitted(0xc001e35860, {0x66596e0?, 0xc000aac0f0?}, {0xc00112fe30?, 0xc003503ce0?}, 0xc003165800?, 0x0?)
/jenkins/workspace/common/system-tests-wrapper/vega/datanode/networkhistory/block_commit_handler.go:77 +0x1a9
code.vegaprotocol.io/vega/datanode/broker.(*SQLStoreBroker).Receive(0xc0002c0c80, {0x66596e0, 0xc000aac0f0})
/jenkins/workspace/common/system-tests-wrapper/vega/datanode/broker/sqlstore_broker.go:139 +0x196
code.vegaprotocol.io/vega/cmd/data-node/commands/start.(*NodeCommand).runNode.func6()
/jenkins/workspace/common/system-tests-wrapper/vega/cmd/data-node/commands/start/node.go:153 +0x70
golang.org/x/sync/errgroup.(*Group).Go.func1()
/jenkins/GOPATH/pkg/mod/golang.org/x/[email protected]/errgroup/errgroup.go:78 +0x56
created by golang.org/x/sync/errgroup.(*Group).Go in goroutine 1
/jenkins/GOPATH/pkg/mod/golang.org/x/[email protected]/errgroup/errgroup.go:75 +0x96
tests.API.grpc_streams_test.test_margin_levels_stream[isolated_margin-0.4]
failed on setup with "AssertionError: Failed to start all scanners. Please check the logs.
assert not True"
Stack trace
request = <SubRequest 'set_up' for <Function test_margin_levels_stream[cross_margin-None]>>
channel_setup = None
context = {'app_state.network_parameters': {'auction.LongBlock': '{"threshold_and_duration":[{"threshold":"24h","duration":"6h"}...41b835d9bfa3a8535a9c221bafc09ab3fe', '513D65AC08': '963228ba16583b826508c6d94c67b9e0abfaab838fdae64a601e33a6d6b81209'}}
@pytest.fixture(scope="session", autouse=True)
def set_up(request, channel_setup, context):
st = f"Setup Session {request.node.name}"
logging.info("*" * len(st))
logging.info(st)
logging.info("*" * len(st))
logging.info(f"using default product: {DEFAULT_PRODUCT}")
session_start_time = time.time()
logging.info(f"Scanner cleanup")
# assert os.path.exists(VEGATOOLS), "Please ensure vegatools is available on the path."
# logging.info(f"Using vegatools found at: {VEGATOOLS}")
kill_vegatools()
clean_eventscanner_dir() # Can only do this once existing processes have been killed
if NO_DATA_NODE_TEST_CASE:
yield
return
> start_scanners()
conftest.py:473:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
conftest.py:270: in start_scanners
check_scanner_for_errors(event_scanner)
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
popen = <Popen: returncode: 1 args: ['/jenkins/workspace/common/system-tests-wrapper...>
def check_scanner_for_errors(popen):
timeout = 10
now = time.time()
# Scanner opens logs for each file type on the fly. If we can find the being block file, that means we're listening
# and will create each subsequent file as required.
eventscanner_path = f"{ROOT_DIR}/eventscanner/"
begin_block_log_file = pathlib.Path(eventscanner_path, "BUS_EVENT_TYPE_BEGIN_BLOCK.txt")
ps = None
logging.info(f"Waiting for begin_block log file to show scanner is running... Waiting up to {timeout} seconds.")
found_log = False
while time.time() <= now + timeout:
ps = get_vega_stream_processes()
if len(ps) < 1:
logging.info(f"Waiting for {1} vega stream processes. Found: {len(ps)}")
time.sleep(0.1)
continue
if begin_block_log_file.exists():
logging.info("Found log file. Scanner is running.")
found_log = True
break
else:
logging.info(f"No log file found at: {begin_block_log_file}.")
# TODO hack, if nullchain make some blocks happen
if NULL_BLOCK_CHAIN:
nullchain.move_forward(10)
time.sleep(0.1)
continue
logging.info(
f"Finished looking for logs and vegatools stream process. Found: {found_log}. Now checking error states of scanner process."
)
# Save this result, but first check all the popens, since their stdout/err gives more useful logging.
failed = False
if popen.poll() is not None:
failed = True
out, err = popen.communicate()
logging.error(f"Error starting scanner: \n{out}\n{err}")
> assert not failed, "Failed to start all scanners. Please check the logs."
E AssertionError: Failed to start all scanners. Please check the logs.
E assert not True
conftest.py:342: AssertionError
-------------------- ERROR FOUND WITH NETWORK. ERROR 1 of 1 --------------------
Trace from file: /jenkins/workspace/common/system-tests-wrapper/networkdata/testnet/logs/testnet-nodeset-full-2-full/data-node-2.stderr-2024-07-09T10:27:10Z.log
panic: failed to snapshot data
goroutine 728 [running]:
go.uber.org/zap/zapcore.CheckWriteAction.OnWrite(0x1?, 0x1?, {0x0?, 0x0?, 0xc00425adc0?})
/jenkins/GOPATH/pkg/mod/go.uber.org/[email protected]/zapcore/entry.go:196 +0x54
go.uber.org/zap/zapcore.(*CheckedEntry).Write(0xc000d08270, {0xc005b3a080, 0x1, 0x1})
/jenkins/GOPATH/pkg/mod/go.uber.org/[email protected]/zapcore/entry.go:262 +0x3ec
go.uber.org/zap.(*Logger).Panic(0xc00436aed0?, {0x5e25a34?, 0xe?}, {0xc005b3a080, 0x1, 0x1})
/jenkins/GOPATH/pkg/mod/go.uber.org/[email protected]/logger.go:258 +0x51
code.vegaprotocol.io/vega/datanode/networkhistory/snapshot.(*Service).createNewSnapshot.func3()
/jenkins/workspace/common/system-tests-wrapper/vega/datanode/networkhistory/snapshot/service_create_snapshot.go:139 +0x29f
code.vegaprotocol.io/vega/datanode/networkhistory/snapshot.(*Service).createNewSnapshot(0xc00119c000, {0x66596e0?, 0xc000aac0f0}, {0xc00112fe30, 0xb}, 0xc00387bf20?, 0x1)
/jenkins/workspace/common/system-tests-wrapper/vega/datanode/networkhistory/snapshot/service_create_snapshot.go:150 +0x7ff
code.vegaprotocol.io/vega/datanode/networkhistory/snapshot.(*Service).CreateSnapshotAsynchronously(...)
/jenkins/workspace/common/system-tests-wrapper/vega/datanode/networkhistory/snapshot/service_create_snapshot.go:54
code.vegaprotocol.io/vega/datanode/networkhistory/snapshot.(*Service).SnapshotData(0x0?, {0x66596e0?, 0xc000aac0f0?}, {0xc00112fe30?, 0x0?}, 0xc0011c5db0?)
/jenkins/workspace/common/system-tests-wrapper/vega/datanode/networkhistory/snapshot/service.go:94 +0x3b
code.vegaprotocol.io/vega/datanode/networkhistory.(*BlockCommitHandler).OnBlockCommitted.func1()
/jenkins/workspace/common/system-tests-wrapper/vega/datanode/networkhistory/block_commit_handler.go:66 +0x3c
github.com/cenkalti/backoff.RetryNotify(0xc003503d98, {0x6640948, 0xc001479a40}, 0x0)
/jenkins/GOPATH/pkg/mod/github.com/cenkalti/[email protected]+incompatible/retry.go:37 +0x164
github.com/cenkalti/backoff.Retry(...)
/jenkins/GOPATH/pkg/mod/github.com/cenkalti/[email protected]+incompatible/retry.go:24
code.vegaprotocol.io/vega/datanode/networkhistory.(*BlockCommitHandler).OnBlockCommitted(0xc001e35860, {0x66596e0?, 0xc000aac0f0?}, {0xc00112fe30?, 0xc003503ce0?}, 0xc003165800?, 0x0?)
/jenkins/workspace/common/system-tests-wrapper/vega/datanode/networkhistory/block_commit_handler.go:77 +0x1a9
code.vegaprotocol.io/vega/datanode/broker.(*SQLStoreBroker).Receive(0xc0002c0c80, {0x66596e0, 0xc000aac0f0})
/jenkins/workspace/common/system-tests-wrapper/vega/datanode/broker/sqlstore_broker.go:139 +0x196
code.vegaprotocol.io/vega/cmd/data-node/commands/start.(*NodeCommand).runNode.func6()
/jenkins/workspace/common/system-tests-wrapper/vega/cmd/data-node/commands/start/node.go:153 +0x70
golang.org/x/sync/errgroup.(*Group).Go.func1()
/jenkins/GOPATH/pkg/mod/golang.org/x/[email protected]/errgroup/errgroup.go:78 +0x56
created by golang.org/x/sync/errgroup.(*Group).Go in goroutine 1
/jenkins/GOPATH/pkg/mod/golang.org/x/[email protected]/errgroup/errgroup.go:75 +0x96
tests.assets.asset_test.test_Withdraw_Enacted_BuiltIn_Asset
failed on setup with "AssertionError: Failed to start all scanners. Please check the logs.
assert not True"
Stack trace
request = <SubRequest 'set_up' for <Function test_margin_levels_stream[cross_margin-None]>>
channel_setup = None
context = {'app_state.network_parameters': {'auction.LongBlock': '{"threshold_and_duration":[{"threshold":"24h","duration":"6h"}...41b835d9bfa3a8535a9c221bafc09ab3fe', '513D65AC08': '963228ba16583b826508c6d94c67b9e0abfaab838fdae64a601e33a6d6b81209'}}
@pytest.fixture(scope="session", autouse=True)
def set_up(request, channel_setup, context):
st = f"Setup Session {request.node.name}"
logging.info("*" * len(st))
logging.info(st)
logging.info("*" * len(st))
logging.info(f"using default product: {DEFAULT_PRODUCT}")
session_start_time = time.time()
logging.info(f"Scanner cleanup")
# assert os.path.exists(VEGATOOLS), "Please ensure vegatools is available on the path."
# logging.info(f"Using vegatools found at: {VEGATOOLS}")
kill_vegatools()
clean_eventscanner_dir() # Can only do this once existing processes have been killed
if NO_DATA_NODE_TEST_CASE:
yield
return
> start_scanners()
conftest.py:473:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
conftest.py:270: in start_scanners
check_scanner_for_errors(event_scanner)
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
popen = <Popen: returncode: 1 args: ['/jenkins/workspace/common/system-tests-wrapper...>
def check_scanner_for_errors(popen):
timeout = 10
now = time.time()
# Scanner opens logs for each file type on the fly. If we can find the being block file, that means we're listening
# and will create each subsequent file as required.
eventscanner_path = f"{ROOT_DIR}/eventscanner/"
begin_block_log_file = pathlib.Path(eventscanner_path, "BUS_EVENT_TYPE_BEGIN_BLOCK.txt")
ps = None
logging.info(f"Waiting for begin_block log file to show scanner is running... Waiting up to {timeout} seconds.")
found_log = False
while time.time() <= now + timeout:
ps = get_vega_stream_processes()
if len(ps) < 1:
logging.info(f"Waiting for {1} vega stream processes. Found: {len(ps)}")
time.sleep(0.1)
continue
if begin_block_log_file.exists():
logging.info("Found log file. Scanner is running.")
found_log = True
break
else:
logging.info(f"No log file found at: {begin_block_log_file}.")
# TODO hack, if nullchain make some blocks happen
if NULL_BLOCK_CHAIN:
nullchain.move_forward(10)
time.sleep(0.1)
continue
logging.info(
f"Finished looking for logs and vegatools stream process. Found: {found_log}. Now checking error states of scanner process."
)
# Save this result, but first check all the popens, since their stdout/err gives more useful logging.
failed = False
if popen.poll() is not None:
failed = True
out, err = popen.communicate()
logging.error(f"Error starting scanner: \n{out}\n{err}")
> assert not failed, "Failed to start all scanners. Please check the logs."
E AssertionError: Failed to start all scanners. Please check the logs.
E assert not True
conftest.py:342: AssertionError
-------------------- ERROR FOUND WITH NETWORK. ERROR 1 of 1 --------------------
Trace from file: /jenkins/workspace/common/system-tests-wrapper/networkdata/testnet/logs/testnet-nodeset-full-2-full/data-node-2.stderr-2024-07-09T10:27:10Z.log
panic: failed to snapshot data
goroutine 728 [running]:
go.uber.org/zap/zapcore.CheckWriteAction.OnWrite(0x1?, 0x1?, {0x0?, 0x0?, 0xc00425adc0?})
/jenkins/GOPATH/pkg/mod/go.uber.org/[email protected]/zapcore/entry.go:196 +0x54
go.uber.org/zap/zapcore.(*CheckedEntry).Write(0xc000d08270, {0xc005b3a080, 0x1, 0x1})
/jenkins/GOPATH/pkg/mod/go.uber.org/[email protected]/zapcore/entry.go:262 +0x3ec
go.uber.org/zap.(*Logger).Panic(0xc00436aed0?, {0x5e25a34?, 0xe?}, {0xc005b3a080, 0x1, 0x1})
/jenkins/GOPATH/pkg/mod/go.uber.org/[email protected]/logger.go:258 +0x51
code.vegaprotocol.io/vega/datanode/networkhistory/snapshot.(*Service).createNewSnapshot.func3()
/jenkins/workspace/common/system-tests-wrapper/vega/datanode/networkhistory/snapshot/service_create_snapshot.go:139 +0x29f
code.vegaprotocol.io/vega/datanode/networkhistory/snapshot.(*Service).createNewSnapshot(0xc00119c000, {0x66596e0?, 0xc000aac0f0}, {0xc00112fe30, 0xb}, 0xc00387bf20?, 0x1)
/jenkins/workspace/common/system-tests-wrapper/vega/datanode/networkhistory/snapshot/service_create_snapshot.go:150 +0x7ff
code.vegaprotocol.io/vega/datanode/networkhistory/snapshot.(*Service).CreateSnapshotAsynchronously(...)
/jenkins/workspace/common/system-tests-wrapper/vega/datanode/networkhistory/snapshot/service_create_snapshot.go:54
code.vegaprotocol.io/vega/datanode/networkhistory/snapshot.(*Service).SnapshotData(0x0?, {0x66596e0?, 0xc000aac0f0?}, {0xc00112fe30?, 0x0?}, 0xc0011c5db0?)
/jenkins/workspace/common/system-tests-wrapper/vega/datanode/networkhistory/snapshot/service.go:94 +0x3b
code.vegaprotocol.io/vega/datanode/networkhistory.(*BlockCommitHandler).OnBlockCommitted.func1()
/jenkins/workspace/common/system-tests-wrapper/vega/datanode/networkhistory/block_commit_handler.go:66 +0x3c
github.com/cenkalti/backoff.RetryNotify(0xc003503d98, {0x6640948, 0xc001479a40}, 0x0)
/jenkins/GOPATH/pkg/mod/github.com/cenkalti/[email protected]+incompatible/retry.go:37 +0x164
github.com/cenkalti/backoff.Retry(...)
/jenkins/GOPATH/pkg/mod/github.com/cenkalti/[email protected]+incompatible/retry.go:24
code.vegaprotocol.io/vega/datanode/networkhistory.(*BlockCommitHandler).OnBlockCommitted(0xc001e35860, {0x66596e0?, 0xc000aac0f0?}, {0xc00112fe30?, 0xc003503ce0?}, 0xc003165800?, 0x0?)
/jenkins/workspace/common/system-tests-wrapper/vega/datanode/networkhistory/block_commit_handler.go:77 +0x1a9
code.vegaprotocol.io/vega/datanode/broker.(*SQLStoreBroker).Receive(0xc0002c0c80, {0x66596e0, 0xc000aac0f0})
/jenkins/workspace/common/system-tests-wrapper/vega/datanode/broker/sqlstore_broker.go:139 +0x196
code.vegaprotocol.io/vega/cmd/data-node/commands/start.(*NodeCommand).runNode.func6()
/jenkins/workspace/common/system-tests-wrapper/vega/cmd/data-node/commands/start/node.go:153 +0x70
golang.org/x/sync/errgroup.(*Group).Go.func1()
/jenkins/GOPATH/pkg/mod/golang.org/x/[email protected]/errgroup/errgroup.go:78 +0x56
created by golang.org/x/sync/errgroup.(*Group).Go in goroutine 1
/jenkins/GOPATH/pkg/mod/golang.org/x/[email protected]/errgroup/errgroup.go:75 +0x96
tests.assets.erc20_asset_test.test_propose_new_erc20_token
failed on setup with "AssertionError: Failed to start all scanners. Please check the logs.
assert not True"
Stack trace
request = <SubRequest 'set_up' for <Function test_margin_levels_stream[cross_margin-None]>>
channel_setup = None
context = {'app_state.network_parameters': {'auction.LongBlock': '{"threshold_and_duration":[{"threshold":"24h","duration":"6h"}...41b835d9bfa3a8535a9c221bafc09ab3fe', '513D65AC08': '963228ba16583b826508c6d94c67b9e0abfaab838fdae64a601e33a6d6b81209'}}
@pytest.fixture(scope="session", autouse=True)
def set_up(request, channel_setup, context):
st = f"Setup Session {request.node.name}"
logging.info("*" * len(st))
logging.info(st)
logging.info("*" * len(st))
logging.info(f"using default product: {DEFAULT_PRODUCT}")
session_start_time = time.time()
logging.info(f"Scanner cleanup")
# assert os.path.exists(VEGATOOLS), "Please ensure vegatools is available on the path."
# logging.info(f"Using vegatools found at: {VEGATOOLS}")
kill_vegatools()
clean_eventscanner_dir() # Can only do this once existing processes have been killed
if NO_DATA_NODE_TEST_CASE:
yield
return
> start_scanners()
conftest.py:473:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
conftest.py:270: in start_scanners
check_scanner_for_errors(event_scanner)
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
popen = <Popen: returncode: 1 args: ['/jenkins/workspace/common/system-tests-wrapper...>
def check_scanner_for_errors(popen):
timeout = 10
now = time.time()
# Scanner opens logs for each file type on the fly. If we can find the being block file, that means we're listening
# and will create each subsequent file as required.
eventscanner_path = f"{ROOT_DIR}/eventscanner/"
begin_block_log_file = pathlib.Path(eventscanner_path, "BUS_EVENT_TYPE_BEGIN_BLOCK.txt")
ps = None
logging.info(f"Waiting for begin_block log file to show scanner is running... Waiting up to {timeout} seconds.")
found_log = False
while time.time() <= now + timeout:
ps = get_vega_stream_processes()
if len(ps) < 1:
logging.info(f"Waiting for {1} vega stream processes. Found: {len(ps)}")
time.sleep(0.1)
continue
if begin_block_log_file.exists():
logging.info("Found log file. Scanner is running.")
found_log = True
break
else:
logging.info(f"No log file found at: {begin_block_log_file}.")
# TODO hack, if nullchain make some blocks happen
if NULL_BLOCK_CHAIN:
nullchain.move_forward(10)
time.sleep(0.1)
continue
logging.info(
f"Finished looking for logs and vegatools stream process. Found: {found_log}. Now checking error states of scanner process."
)
# Save this result, but first check all the popens, since their stdout/err gives more useful logging.
failed = False
if popen.poll() is not None:
failed = True
out, err = popen.communicate()
logging.error(f"Error starting scanner: \n{out}\n{err}")
> assert not failed, "Failed to start all scanners. Please check the logs."
E AssertionError: Failed to start all scanners. Please check the logs.
E assert not True
conftest.py:342: AssertionError
-------------------- ERROR FOUND WITH NETWORK. ERROR 1 of 1 --------------------
Trace from file: /jenkins/workspace/common/system-tests-wrapper/networkdata/testnet/logs/testnet-nodeset-full-2-full/data-node-2.stderr-2024-07-09T10:27:10Z.log
panic: failed to snapshot data
goroutine 728 [running]:
go.uber.org/zap/zapcore.CheckWriteAction.OnWrite(0x1?, 0x1?, {0x0?, 0x0?, 0xc00425adc0?})
/jenkins/GOPATH/pkg/mod/go.uber.org/[email protected]/zapcore/entry.go:196 +0x54
go.uber.org/zap/zapcore.(*CheckedEntry).Write(0xc000d08270, {0xc005b3a080, 0x1, 0x1})
/jenkins/GOPATH/pkg/mod/go.uber.org/[email protected]/zapcore/entry.go:262 +0x3ec
go.uber.org/zap.(*Logger).Panic(0xc00436aed0?, {0x5e25a34?, 0xe?}, {0xc005b3a080, 0x1, 0x1})
/jenkins/GOPATH/pkg/mod/go.uber.org/[email protected]/logger.go:258 +0x51
code.vegaprotocol.io/vega/datanode/networkhistory/snapshot.(*Service).createNewSnapshot.func3()
/jenkins/workspace/common/system-tests-wrapper/vega/datanode/networkhistory/snapshot/service_create_snapshot.go:139 +0x29f
code.vegaprotocol.io/vega/datanode/networkhistory/snapshot.(*Service).createNewSnapshot(0xc00119c000, {0x66596e0?, 0xc000aac0f0}, {0xc00112fe30, 0xb}, 0xc00387bf20?, 0x1)
/jenkins/workspace/common/system-tests-wrapper/vega/datanode/networkhistory/snapshot/service_create_snapshot.go:150 +0x7ff
code.vegaprotocol.io/vega/datanode/networkhistory/snapshot.(*Service).CreateSnapshotAsynchronously(...)
/jenkins/workspace/common/system-tests-wrapper/vega/datanode/networkhistory/snapshot/service_create_snapshot.go:54
code.vegaprotocol.io/vega/datanode/networkhistory/snapshot.(*Service).SnapshotData(0x0?, {0x66596e0?, 0xc000aac0f0?}, {0xc00112fe30?, 0x0?}, 0xc0011c5db0?)
/jenkins/workspace/common/system-tests-wrapper/vega/datanode/networkhistory/snapshot/service.go:94 +0x3b
code.vegaprotocol.io/vega/datanode/networkhistory.(*BlockCommitHandler).OnBlockCommitted.func1()
/jenkins/workspace/common/system-tests-wrapper/vega/datanode/networkhistory/block_commit_handler.go:66 +0x3c
github.com/cenkalti/backoff.RetryNotify(0xc003503d98, {0x6640948, 0xc001479a40}, 0x0)
/jenkins/GOPATH/pkg/mod/github.com/cenkalti/[email protected]+incompatible/retry.go:37 +0x164
github.com/cenkalti/backoff.Retry(...)
/jenkins/GOPATH/pkg/mod/github.com/cenkalti/[email protected]+incompatible/retry.go:24
code.vegaprotocol.io/vega/datanode/networkhistory.(*BlockCommitHandler).OnBlockCommitted(0xc001e35860, {0x66596e0?, 0xc000aac0f0?}, {0xc00112fe30?, 0xc003503ce0?}, 0xc003165800?, 0x0?)
/jenkins/workspace/common/system-tests-wrapper/vega/datanode/networkhistory/block_commit_handler.go:77 +0x1a9
code.vegaprotocol.io/vega/datanode/broker.(*SQLStoreBroker).Receive(0xc0002c0c80, {0x66596e0, 0xc000aac0f0})
/jenkins/workspace/common/system-tests-wrapper/vega/datanode/broker/sqlstore_broker.go:139 +0x196
code.vegaprotocol.io/vega/cmd/data-node/commands/start.(*NodeCommand).runNode.func6()
/jenkins/workspace/common/system-tests-wrapper/vega/cmd/data-node/commands/start/node.go:153 +0x70
golang.org/x/sync/errgroup.(*Group).Go.func1()
/jenkins/GOPATH/pkg/mod/golang.org/x/[email protected]/errgroup/errgroup.go:78 +0x56
created by golang.org/x/sync/errgroup.(*Group).Go in goroutine 1
/jenkins/GOPATH/pkg/mod/golang.org/x/[email protected]/errgroup/errgroup.go:75 +0x96
tests.assets.erc20_asset_test.test_propose_new_erc20_token_on_second_bridge
failed on setup with "AssertionError: Failed to start all scanners. Please check the logs.
assert not True"
Stack trace
request = <SubRequest 'set_up' for <Function test_margin_levels_stream[cross_margin-None]>>
channel_setup = None
context = {'app_state.network_parameters': {'auction.LongBlock': '{"threshold_and_duration":[{"threshold":"24h","duration":"6h"}...41b835d9bfa3a8535a9c221bafc09ab3fe', '513D65AC08': '963228ba16583b826508c6d94c67b9e0abfaab838fdae64a601e33a6d6b81209'}}
@pytest.fixture(scope="session", autouse=True)
def set_up(request, channel_setup, context):
st = f"Setup Session {request.node.name}"
logging.info("*" * len(st))
logging.info(st)
logging.info("*" * len(st))
logging.info(f"using default product: {DEFAULT_PRODUCT}")
session_start_time = time.time()
logging.info(f"Scanner cleanup")
# assert os.path.exists(VEGATOOLS), "Please ensure vegatools is available on the path."
# logging.info(f"Using vegatools found at: {VEGATOOLS}")
kill_vegatools()
clean_eventscanner_dir() # Can only do this once existing processes have been killed
if NO_DATA_NODE_TEST_CASE:
yield
return
> start_scanners()
conftest.py:473:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
conftest.py:270: in start_scanners
check_scanner_for_errors(event_scanner)
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
popen = <Popen: returncode: 1 args: ['/jenkins/workspace/common/system-tests-wrapper...>
def check_scanner_for_errors(popen):
timeout = 10
now = time.time()
# Scanner opens logs for each file type on the fly. If we can find the being block file, that means we're listening
# and will create each subsequent file as required.
eventscanner_path = f"{ROOT_DIR}/eventscanner/"
begin_block_log_file = pathlib.Path(eventscanner_path, "BUS_EVENT_TYPE_BEGIN_BLOCK.txt")
ps = None
logging.info(f"Waiting for begin_block log file to show scanner is running... Waiting up to {timeout} seconds.")
found_log = False
while time.time() <= now + timeout:
ps = get_vega_stream_processes()
if len(ps) < 1:
logging.info(f"Waiting for {1} vega stream processes. Found: {len(ps)}")
time.sleep(0.1)
continue
if begin_block_log_file.exists():
logging.info("Found log file. Scanner is running.")
found_log = True
break
else:
logging.info(f"No log file found at: {begin_block_log_file}.")
# TODO hack, if nullchain make some blocks happen
if NULL_BLOCK_CHAIN:
nullchain.move_forward(10)
time.sleep(0.1)
continue
logging.info(
f"Finished looking for logs and vegatools stream process. Found: {found_log}. Now checking error states of scanner process."
)
# Save this result, but first check all the popens, since their stdout/err gives more useful logging.
failed = False
if popen.poll() is not None:
failed = True
out, err = popen.communicate()
logging.error(f"Error starting scanner: \n{out}\n{err}")
> assert not failed, "Failed to start all scanners. Please check the logs."
E AssertionError: Failed to start all scanners. Please check the logs.
E assert not True
conftest.py:342: AssertionError
-------------------- ERROR FOUND WITH NETWORK. ERROR 1 of 1 --------------------
Trace from file: /jenkins/workspace/common/system-tests-wrapper/networkdata/testnet/logs/testnet-nodeset-full-2-full/data-node-2.stderr-2024-07-09T10:27:10Z.log
panic: failed to snapshot data
goroutine 728 [running]:
go.uber.org/zap/zapcore.CheckWriteAction.OnWrite(0x1?, 0x1?, {0x0?, 0x0?, 0xc00425adc0?})
/jenkins/GOPATH/pkg/mod/go.uber.org/[email protected]/zapcore/entry.go:196 +0x54
go.uber.org/zap/zapcore.(*CheckedEntry).Write(0xc000d08270, {0xc005b3a080, 0x1, 0x1})
/jenkins/GOPATH/pkg/mod/go.uber.org/[email protected]/zapcore/entry.go:262 +0x3ec
go.uber.org/zap.(*Logger).Panic(0xc00436aed0?, {0x5e25a34?, 0xe?}, {0xc005b3a080, 0x1, 0x1})
/jenkins/GOPATH/pkg/mod/go.uber.org/[email protected]/logger.go:258 +0x51
code.vegaprotocol.io/vega/datanode/networkhistory/snapshot.(*Service).createNewSnapshot.func3()
/jenkins/workspace/common/system-tests-wrapper/vega/datanode/networkhistory/snapshot/service_create_snapshot.go:139 +0x29f
code.vegaprotocol.io/vega/datanode/networkhistory/snapshot.(*Service).createNewSnapshot(0xc00119c000, {0x66596e0?, 0xc000aac0f0}, {0xc00112fe30, 0xb}, 0xc00387bf20?, 0x1)
/jenkins/workspace/common/system-tests-wrapper/vega/datanode/networkhistory/snapshot/service_create_snapshot.go:150 +0x7ff
code.vegaprotocol.io/vega/datanode/networkhistory/snapshot.(*Service).CreateSnapshotAsynchronously(...)
/jenkins/workspace/common/system-tests-wrapper/vega/datanode/networkhistory/snapshot/service_create_snapshot.go:54
code.vegaprotocol.io/vega/datanode/networkhistory/snapshot.(*Service).SnapshotData(0x0?, {0x66596e0?, 0xc000aac0f0?}, {0xc00112fe30?, 0x0?}, 0xc0011c5db0?)
/jenkins/workspace/common/system-tests-wrapper/vega/datanode/networkhistory/snapshot/service.go:94 +0x3b
code.vegaprotocol.io/vega/datanode/networkhistory.(*BlockCommitHandler).OnBlockCommitted.func1()
/jenkins/workspace/common/system-tests-wrapper/vega/datanode/networkhistory/block_commit_handler.go:66 +0x3c
github.com/cenkalti/backoff.RetryNotify(0xc003503d98, {0x6640948, 0xc001479a40}, 0x0)
/jenkins/GOPATH/pkg/mod/github.com/cenkalti/[email protected]+incompatible/retry.go:37 +0x164
github.com/cenkalti/backoff.Retry(...)
/jenkins/GOPATH/pkg/mod/github.com/cenkalti/[email protected]+incompatible/retry.go:24
code.vegaprotocol.io/vega/datanode/networkhistory.(*BlockCommitHandler).OnBlockCommitted(0xc001e35860, {0x66596e0?, 0xc000aac0f0?}, {0xc00112fe30?, 0xc003503ce0?}, 0xc003165800?, 0x0?)
/jenkins/workspace/common/system-tests-wrapper/vega/datanode/networkhistory/block_commit_handler.go:77 +0x1a9
code.vegaprotocol.io/vega/datanode/broker.(*SQLStoreBroker).Receive(0xc0002c0c80, {0x66596e0, 0xc000aac0f0})
/jenkins/workspace/common/system-tests-wrapper/vega/datanode/broker/sqlstore_broker.go:139 +0x196
code.vegaprotocol.io/vega/cmd/data-node/commands/start.(*NodeCommand).runNode.func6()
/jenkins/workspace/common/system-tests-wrapper/vega/cmd/data-node/commands/start/node.go:153 +0x70
golang.org/x/sync/errgroup.(*Group).Go.func1()
/jenkins/GOPATH/pkg/mod/golang.org/x/[email protected]/errgroup/errgroup.go:78 +0x56
created by golang.org/x/sync/errgroup.(*Group).Go in goroutine 1
/jenkins/GOPATH/pkg/mod/golang.org/x/[email protected]/errgroup/errgroup.go:75 +0x96
tests.assets.erc20_asset_test.test_modify_erc20_asset[False]
failed on setup with "AssertionError: Failed to start all scanners. Please check the logs.
assert not True"
Stack trace
request = <SubRequest 'set_up' for <Function test_margin_levels_stream[cross_margin-None]>>
channel_setup = None
context = {'app_state.network_parameters': {'auction.LongBlock': '{"threshold_and_duration":[{"threshold":"24h","duration":"6h"}...41b835d9bfa3a8535a9c221bafc09ab3fe', '513D65AC08': '963228ba16583b826508c6d94c67b9e0abfaab838fdae64a601e33a6d6b81209'}}
@pytest.fixture(scope="session", autouse=True)
def set_up(request, channel_setup, context):
st = f"Setup Session {request.node.name}"
logging.info("*" * len(st))
logging.info(st)
logging.info("*" * len(st))
logging.info(f"using default product: {DEFAULT_PRODUCT}")
session_start_time = time.time()
logging.info(f"Scanner cleanup")
# assert os.path.exists(VEGATOOLS), "Please ensure vegatools is available on the path."
# logging.info(f"Using vegatools found at: {VEGATOOLS}")
kill_vegatools()
clean_eventscanner_dir() # Can only do this once existing processes have been killed
if NO_DATA_NODE_TEST_CASE:
yield
return
> start_scanners()
conftest.py:473:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
conftest.py:270: in start_scanners
check_scanner_for_errors(event_scanner)
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
popen = <Popen: returncode: 1 args: ['/jenkins/workspace/common/system-tests-wrapper...>
def check_scanner_for_errors(popen):
timeout = 10
now = time.time()
# Scanner opens logs for each file type on the fly. If we can find the being block file, that means we're listening
# and will create each subsequent file as required.
eventscanner_path = f"{ROOT_DIR}/eventscanner/"
begin_block_log_file = pathlib.Path(eventscanner_path, "BUS_EVENT_TYPE_BEGIN_BLOCK.txt")
ps = None
logging.info(f"Waiting for begin_block log file to show scanner is running... Waiting up to {timeout} seconds.")
found_log = False
while time.time() <= now + timeout:
ps = get_vega_stream_processes()
if len(ps) < 1:
logging.info(f"Waiting for {1} vega stream processes. Found: {len(ps)}")
time.sleep(0.1)
continue
if begin_block_log_file.exists():
logging.info("Found log file. Scanner is running.")
found_log = True
break
else:
logging.info(f"No log file found at: {begin_block_log_file}.")
# TODO hack, if nullchain make some blocks happen
if NULL_BLOCK_CHAIN:
nullchain.move_forward(10)
time.sleep(0.1)
continue
logging.info(
f"Finished looking for logs and vegatools stream process. Found: {found_log}. Now checking error states of scanner process."
)
# Save this result, but first check all the popens, since their stdout/err gives more useful logging.
failed = False
if popen.poll() is not None:
failed = True
out, err = popen.communicate()
logging.error(f"Error starting scanner: \n{out}\n{err}")
> assert not failed, "Failed to start all scanners. Please check the logs."
E AssertionError: Failed to start all scanners. Please check the logs.
E assert not True
conftest.py:342: AssertionError
-------------------- ERROR FOUND WITH NETWORK. ERROR 1 of 1 --------------------
Trace from file: /jenkins/workspace/common/system-tests-wrapper/networkdata/testnet/logs/testnet-nodeset-full-2-full/data-node-2.stderr-2024-07-09T10:27:10Z.log
panic: failed to snapshot data
goroutine 728 [running]:
go.uber.org/zap/zapcore.CheckWriteAction.OnWrite(0x1?, 0x1?, {0x0?, 0x0?, 0xc00425adc0?})
/jenkins/GOPATH/pkg/mod/go.uber.org/[email protected]/zapcore/entry.go:196 +0x54
go.uber.org/zap/zapcore.(*CheckedEntry).Write(0xc000d08270, {0xc005b3a080, 0x1, 0x1})
/jenkins/GOPATH/pkg/mod/go.uber.org/[email protected]/zapcore/entry.go:262 +0x3ec
go.uber.org/zap.(*Logger).Panic(0xc00436aed0?, {0x5e25a34?, 0xe?}, {0xc005b3a080, 0x1, 0x1})
/jenkins/GOPATH/pkg/mod/go.uber.org/[email protected]/logger.go:258 +0x51
code.vegaprotocol.io/vega/datanode/networkhistory/snapshot.(*Service).createNewSnapshot.func3()
/jenkins/workspace/common/system-tests-wrapper/vega/datanode/networkhistory/snapshot/service_create_snapshot.go:139 +0x29f
code.vegaprotocol.io/vega/datanode/networkhistory/snapshot.(*Service).createNewSnapshot(0xc00119c000, {0x66596e0?, 0xc000aac0f0}, {0xc00112fe30, 0xb}, 0xc00387bf20?, 0x1)
/jenkins/workspace/common/system-tests-wrapper/vega/datanode/networkhistory/snapshot/service_create_snapshot.go:150 +0x7ff
code.vegaprotocol.io/vega/datanode/networkhistory/snapshot.(*Service).CreateSnapshotAsynchronously(...)
/jenkins/workspace/common/system-tests-wrapper/vega/datanode/networkhistory/snapshot/service_create_snapshot.go:54
code.vegaprotocol.io/vega/datanode/networkhistory/snapshot.(*Service).SnapshotData(0x0?, {0x66596e0?, 0xc000aac0f0?}, {0xc00112fe30?, 0x0?}, 0xc0011c5db0?)
/jenkins/workspace/common/system-tests-wrapper/vega/datanode/networkhistory/snapshot/service.go:94 +0x3b
code.vegaprotocol.io/vega/datanode/networkhistory.(*BlockCommitHandler).OnBlockCommitted.func1()
/jenkins/workspace/common/system-tests-wrapper/vega/datanode/networkhistory/block_commit_handler.go:66 +0x3c
github.com/cenkalti/backoff.RetryNotify(0xc003503d98, {0x6640948, 0xc001479a40}, 0x0)
/jenkins/GOPATH/pkg/mod/github.com/cenkalti/[email protected]+incompatible/retry.go:37 +0x164
github.com/cenkalti/backoff.Retry(...)
/jenkins/GOPATH/pkg/mod/github.com/cenkalti/[email protected]+incompatible/retry.go:24
code.vegaprotocol.io/vega/datanode/networkhistory.(*BlockCommitHandler).OnBlockCommitted(0xc001e35860, {0x66596e0?, 0xc000aac0f0?}, {0xc00112fe30?, 0xc003503ce0?}, 0xc003165800?, 0x0?)
/jenkins/workspace/common/system-tests-wrapper/vega/datanode/networkhistory/block_commit_handler.go:77 +0x1a9
code.vegaprotocol.io/vega/datanode/broker.(*SQLStoreBroker).Receive(0xc0002c0c80, {0x66596e0, 0xc000aac0f0})
/jenkins/workspace/common/system-tests-wrapper/vega/datanode/broker/sqlstore_broker.go:139 +0x196
code.vegaprotocol.io/vega/cmd/data-node/commands/start.(*NodeCommand).runNode.func6()
/jenkins/workspace/common/system-tests-wrapper/vega/cmd/data-node/commands/start/node.go:153 +0x70
golang.org/x/sync/errgroup.(*Group).Go.func1()
/jenkins/GOPATH/pkg/mod/golang.org/x/[email protected]/errgroup/errgroup.go:78 +0x56
created by golang.org/x/sync/errgroup.(*Group).Go in goroutine 1
/jenkins/GOPATH/pkg/mod/golang.org/x/[email protected]/errgroup/errgroup.go:75 +0x96
more test results are not shown here, view them on Jenkins
Loading