From c051ecc4741757f6953ec6680a64ce8b2c498c9f Mon Sep 17 00:00:00 2001 From: Raymond Sukanto Date: Fri, 30 Aug 2024 02:13:23 +0700 Subject: [PATCH] devnet --- avalanche/network.go | 12 ++ constants/constants.go | 4 + node/devnet.go | 322 +++++++++++++++++++++++++++++++++++++++++ node/node.go | 58 ++++++++ node/ssh.go | 40 +++++ 5 files changed, 436 insertions(+) create mode 100644 node/devnet.go diff --git a/avalanche/network.go b/avalanche/network.go index c996520..7c576c8 100644 --- a/avalanche/network.go +++ b/avalanche/network.go @@ -18,6 +18,8 @@ const ( Mainnet Fuji Devnet + DevnetAPIEndpoint = "" + DevnetNetworkID = 1338 ) const ( @@ -82,6 +84,16 @@ func MainnetNetwork() Network { return NewNetwork(Mainnet, constants.MainnetID, MainnetAPIEndpoint) } +func DevnetNetwork(endpoint string, id uint32) Network { + if endpoint == "" { + endpoint = DevnetAPIEndpoint + } + if id == 0 { + id = DevnetNetworkID + } + return NewNetwork(Devnet, id, endpoint) +} + func (n Network) GenesisParams() *genesis.Params { switch n.Kind { case Devnet: diff --git a/constants/constants.go b/constants/constants.go index 70e8ee4..2e92439 100644 --- a/constants/constants.go +++ b/constants/constants.go @@ -42,7 +42,9 @@ const ( SSHFileOpsTimeout = 100 * time.Second SSHPOSTTimeout = 10 * time.Second SSHScriptTimeout = 2 * time.Minute + SSHDirOpsTimeout = 10 * time.Second RemoteHostUser = "ubuntu" + DockerNodeConfigPath = "/.avalanchego/configs/" // node CloudNodeCLIConfigBasePath = "/home/ubuntu/.avalanche-cli/" @@ -50,6 +52,8 @@ const ( CloudNodeConfigPath = "/home/ubuntu/.avalanchego/configs/" ServicesDir = "services" DashboardsDir = "dashboards" + GenesisFileName = "genesis.json" + NodeFileName = "node.json" // services ServiceAvalanchego = "avalanchego" ServicePromtail = "promtail" diff --git a/node/devnet.go b/node/devnet.go new file mode 100644 index 0000000..e52d617 --- /dev/null +++ b/node/devnet.go @@ -0,0 +1,322 @@ +// Copyright (C) 2024, Ava Labs, Inc. All rights reserved. +// See the file LICENSE for licensing terms. + +package node + +import ( + "encoding/json" + "fmt" + "path/filepath" + "slices" + "strconv" + "strings" + "sync" + "time" + + "github.com/ava-labs/avalanche-tooling-sdk-go/avalanche" + "github.com/ava-labs/avalanche-tooling-sdk-go/key" + "github.com/ava-labs/avalanchego/config" + "github.com/ava-labs/avalanchego/utils/crypto/bls" + "github.com/ava-labs/avalanchego/utils/formatting" + "github.com/ava-labs/avalanchego/utils/logging" + "github.com/ava-labs/avalanchego/vms/platformvm/signer" + "golang.org/x/exp/maps" + + "github.com/ava-labs/avalanche-tooling-sdk-go/constants" + "github.com/ava-labs/avalanche-tooling-sdk-go/utils" + coreth_params "github.com/ava-labs/coreth/params" +) + +// difference between unlock schedule locktime and startime in original genesis +const ( + genesisLocktimeStartimeDelta = 2836800 + hexa0Str = "0x0" + defaultLocalCChainFundedAddress = "8db97C7cEcE249c2b98bDC0226Cc4C2A57BF52FC" + defaultLocalCChainFundedBalance = "0x295BE96E64066972000000" + allocationCommonEthAddress = "0xb3d82b1367d362de99ab59a658165aff520cbd4d" +) + +// func SetupDevnet(clusterName string, hosts []*models.Host, apiNodeIPMap map[string]string) error { +// func SetupDevnet(clusterName string, hosts []*Node, apiNodeIPMap map[string]string) error { +func SetupDevnet(hosts []*Node, apiNodeIPMap map[string]string) error { + //ansibleHosts, err := ansible.GetHostMapfromAnsibleInventory(app.GetAnsibleInventoryDirPath(clusterName)) + //if err != nil { + // return err + //} + + // set devnet network + endpointIP := "" + if len(apiNodeIPMap) > 0 { + endpointIP = maps.Values(apiNodeIPMap)[0] + } else { + //endpointIP = ansibleHosts[ansibleHostIDs[0]].IP + endpointIP = hosts[0].IP + } + endpoint := fmt.Sprintf("http://%s:%d", endpointIP, constants.AvalanchegoAPIPort) + network := avalanche.DevnetNetwork(endpoint, 0) + // network = models.NewNetworkFromCluster(network, clusterName) + + // get random staking key for devnet genesis + k, err := key.NewSoft() + if err != nil { + return err + } + // stakingAddrStr := k.X()[0] + stakingAddrStr, err := k.X(network.HRP()) + if err != nil { + return err + } + + // get ewoq key as funded key for devnet genesis + // k, err = key.LoadEwoq(network.ID) + k, err = key.LoadEwoq() + if err != nil { + return err + } + // walletAddrStr := k.X()[0] + walletAddrStr, err := k.X(network.HRP()) + if err != nil { + return err + } + + for _, host := range hosts { + host.CreateStakingFiles() + host.SetNodeID() + } + + // exclude API nodes from genesis file generation as they will have no stake + hostsAPI := utils.Filter(hosts, func(h *Node) bool { + return slices.Contains(maps.Keys(apiNodeIPMap), h.GetCloudID()) + }) + hostsWithoutAPI := utils.Filter(hosts, func(h *Node) bool { + return !slices.Contains(maps.Keys(apiNodeIPMap), h.GetCloudID()) + }) + hostsWithoutAPIIDs := utils.Map(hostsWithoutAPI, func(h *Node) string { return h.NodeID }) + + // create genesis file at each node dir + genesisBytes, err := generateCustomGenesis(network.ID, walletAddrStr, stakingAddrStr, hostsWithoutAPI) + if err != nil { + return err + } + // make sure that custom genesis is saved to the subnet dir + //if err := os.WriteFile(app.GetGenesisPath(subnetName), genesisBytes, constants.WriteReadReadPerms); err != nil { + // return err + //} + + // create avalanchego conf node.json at each node dir + bootstrapIPs := []string{} + bootstrapIDs := []string{} + // append makes sure that hostsWithoutAPI i.e. validators are proccessed first and API nodes will have full list of validators to bootstrap + for _, host := range append(hostsWithoutAPI, hostsAPI...) { + confMap := map[string]interface{}{} + confMap[config.HTTPHostKey] = "" + confMap[config.PublicIPKey] = host.IP + confMap[config.NetworkNameKey] = fmt.Sprintf("network-%d", network.ID) + confMap[config.BootstrapIDsKey] = strings.Join(bootstrapIDs, ",") + confMap[config.BootstrapIPsKey] = strings.Join(bootstrapIPs, ",") + confMap[config.GenesisFileKey] = filepath.Join(constants.DockerNodeConfigPath, "genesis.json") + confBytes, err := json.MarshalIndent(confMap, "", " ") + if err != nil { + return err + } + + host.SetDevnetInfo(genesisBytes, confBytes) + //if err := os.WriteFile(filepath.Join(app.GetNodeInstanceDirPath(host.GetCloudID()), "genesis.json"), genesisBytes, constants.WriteReadReadPerms); err != nil { + // return err + //} + //if err := os.WriteFile(filepath.Join(app.GetNodeInstanceDirPath(host.GetCloudID()), "node.json"), confBytes, constants.WriteReadReadPerms); err != nil { + // return err + //} + if slices.Contains(hostsWithoutAPIIDs, host.NodeID) { + //nodeID, err := getNodeID(app.GetNodeInstanceDirPath(host.GetCloudID())) + //if err != nil { + // return err + //} + //bootstrapIDs = append(bootstrapIDs, nodeID.String()) + bootstrapIDs = append(bootstrapIDs, host.NodeID) + bootstrapIPs = append(bootstrapIPs, fmt.Sprintf("%s:9651", host.IP)) + } + } + // update node/s genesis + conf and start + wg := sync.WaitGroup{} + wgResults := NodeResults{} + for _, host := range hosts { + wg.Add(1) + go func(nodeResults *NodeResults, host *Node) { + defer wg.Done() + + // keyPath := filepath.Join(app.GetNodesDir(), host.GetCloudID()) + if err := host.RunSSHSetupDevNet(); err != nil { + nodeResults.AddResult(host.NodeID, nil, err) + // ux.Logger.RedXToUser(utils.ScriptLog(host.NodeID, "Setup devnet err: %v", err)) + return + } + // ux.Logger.GreenCheckmarkToUser(utils.ScriptLog(host.NodeID, "Setup devnet")) + }(&wgResults, host) + } + wg.Wait() + // ux.Logger.PrintLineSeparator() + for _, node := range hosts { + if wgResults.HasNodeIDWithError(node.NodeID) { + // ux.Logger.RedXToUser("Node %s is ERROR with error: %s", node.NodeID, wgResults.GetErrorHostMap()[node.NodeID]) + fmt.Printf("Node %s is ERROR with error: %s", node.NodeID, wgResults.GetErrorHostMap()[node.NodeID]) + } else { + //nodeID, err := getNodeID(app.GetNodeInstanceDirPath(node.GetCloudID())) + //if err != nil { + // return err + //} + //ux.Logger.GreenCheckmarkToUser("Node %s[%s] is SETUP as devnet", node.GetCloudID(), nodeID) + fmt.Printf("Node %s[%s] is SETUP as devnet", node.GetCloudID(), node.NodeID) + } + } + // stop execution if at least one node failed + if wgResults.HasErrors() { + return fmt.Errorf("failed to deploy node(s) %s", wgResults.GetErrorHostMap()) + } + // ux.Logger.PrintLineSeparator() + // ux.Logger.PrintToUser("Devnet Network Id: %s", logging.Green.Wrap(strconv.FormatUint(uint64(network.ID), 10))) + // ux.Logger.PrintToUser("Devnet Endpoint: %s", logging.Green.Wrap(network.Endpoint)) + // ux.Logger.PrintLineSeparator() + fmt.Printf("Devnet Network Id: %s", logging.Green.Wrap(strconv.FormatUint(uint64(network.ID), 10))) + fmt.Printf("Devnet Endpoint: %s", logging.Green.Wrap(network.Endpoint)) + // update cluster config with network information + //clustersConfig, err := app.LoadClustersConfig() + //if err != nil { + // return err + //} + //clusterConfig := clustersConfig.Clusters[clusterName] + //clusterConfig.Network = network + //clustersConfig.Clusters[clusterName] = clusterConfig + //return app.WriteClustersConfigFile(&clustersConfig) + return nil +} + +func generateCustomGenesis( + networkID uint32, + walletAddr string, + stakingAddr string, + hosts []*Node, +) ([]byte, error) { + genesisMap := map[string]interface{}{} + + // cchain + cChainGenesisBytes, err := generateCustomCchainGenesis() + if err != nil { + return nil, err + } + genesisMap["cChainGenesis"] = string(cChainGenesisBytes) + + // pchain genesis + genesisMap["networkID"] = networkID + startTime := time.Now().Unix() + genesisMap["startTime"] = startTime + initialStakers := []map[string]interface{}{} + for _, host := range hosts { + //nodeDirPath := app.GetNodeInstanceDirPath(host.GetCloudID()) + //blsPath := filepath.Join(nodeDirPath, constants.BLSKeyFileName) + //blsKey, err := os.ReadFile(blsPath) + //if err != nil { + // return nil, err + //} + blsSk, err := bls.SecretKeyFromBytes(host.StakingFiles.BlsSignerKeyBytes) + if err != nil { + return nil, err + } + p := signer.NewProofOfPossession(blsSk) + pk, err := formatting.Encode(formatting.HexNC, p.PublicKey[:]) + if err != nil { + return nil, err + } + pop, err := formatting.Encode(formatting.HexNC, p.ProofOfPossession[:]) + if err != nil { + return nil, err + } + //nodeID, err := getNodeID(nodeDirPath) + //if err != nil { + // return nil, err + //} + //initialStaker := map[string]interface{}{ + // "nodeID": nodeID, + // "rewardAddress": walletAddr, + // "delegationFee": 1000000, + // "signer": map[string]interface{}{ + // "proofOfPossession": pop, + // "publicKey": pk, + // }, + //} + initialStaker := map[string]interface{}{ + "nodeID": host.NodeID, + "rewardAddress": walletAddr, + "delegationFee": 1000000, + "signer": map[string]interface{}{ + "proofOfPossession": pop, + "publicKey": pk, + }, + } + initialStakers = append(initialStakers, initialStaker) + } + genesisMap["initialStakeDuration"] = 31536000 + genesisMap["initialStakeDurationOffset"] = 5400 + genesisMap["initialStakers"] = initialStakers + lockTime := startTime + genesisLocktimeStartimeDelta + allocations := []interface{}{} + alloc := map[string]interface{}{ + "avaxAddr": walletAddr, + "ethAddr": allocationCommonEthAddress, + "initialAmount": 300000000000000000, + "unlockSchedule": []interface{}{ + map[string]interface{}{"amount": 20000000000000000}, + map[string]interface{}{"amount": 10000000000000000, "locktime": lockTime}, + }, + } + allocations = append(allocations, alloc) + alloc = map[string]interface{}{ + "avaxAddr": stakingAddr, + "ethAddr": allocationCommonEthAddress, + "initialAmount": 0, + "unlockSchedule": []interface{}{ + map[string]interface{}{"amount": 10000000000000000, "locktime": lockTime}, + }, + } + allocations = append(allocations, alloc) + genesisMap["allocations"] = allocations + genesisMap["initialStakedFunds"] = []interface{}{ + stakingAddr, + } + genesisMap["message"] = "{{ fun_quote }}" + + return json.MarshalIndent(genesisMap, "", " ") +} + +func generateCustomCchainGenesis() ([]byte, error) { + cChainGenesisMap := map[string]interface{}{} + cChainGenesisMap["config"] = coreth_params.AvalancheLocalChainConfig + cChainGenesisMap["nonce"] = hexa0Str + cChainGenesisMap["timestamp"] = hexa0Str + cChainGenesisMap["extraData"] = "0x00" + cChainGenesisMap["gasLimit"] = "0x5f5e100" + cChainGenesisMap["difficulty"] = hexa0Str + cChainGenesisMap["mixHash"] = "0x0000000000000000000000000000000000000000000000000000000000000000" + cChainGenesisMap["coinbase"] = "0x0000000000000000000000000000000000000000" + cChainGenesisMap["alloc"] = map[string]interface{}{ + defaultLocalCChainFundedAddress: map[string]interface{}{ + "balance": defaultLocalCChainFundedBalance, + }, + } + cChainGenesisMap["number"] = hexa0Str + cChainGenesisMap["gasUsed"] = hexa0Str + cChainGenesisMap["parentHash"] = "0x0000000000000000000000000000000000000000000000000000000000000000" + return json.Marshal(cChainGenesisMap) +} + +//func getNodeID(nodeDir string) (ids.NodeID, error) { +// certBytes, err := os.ReadFile(filepath.Join(nodeDir, constants.StakerCertFileName)) +// if err != nil { +// return ids.EmptyNodeID, err +// } +// nodeID, err := utils.ToNodeID(certBytes) +// if err != nil { +// return ids.EmptyNodeID, err +// } +// return nodeID, nil +//} diff --git a/node/node.go b/node/node.go index 1007e05..a725bf8 100644 --- a/node/node.go +++ b/node/node.go @@ -8,6 +8,7 @@ import ( "bytes" "context" "fmt" + "github.com/ava-labs/avalanchego/staking" "io" "net" "net/http" @@ -42,6 +43,22 @@ type SSHConfig struct { Params map[string]string // additional parameters to pass to the ssh command } +// TODO: Rename this +type StakingFiles struct { + CertBytes []byte + + KeyBytes []byte + + BlsSignerKeyBytes []byte +} + +// TODO: Rename this +type DevNetInfo struct { + GenesisBytes []byte + + NodeConfigBytes []byte +} + // Node is an output of CreateNodes type Node struct { // NodeID is Avalanche Node ID of the node @@ -81,6 +98,11 @@ type Node struct { // BLS provides a way to aggregate signatures off chain into a single signature that can be efficiently verified on chain. // For more information about how BLS is used on the P-Chain, please head to https://docs.avax.network/cross-chain/avalanche-warp-messaging/deep-dive#bls-multi-signatures-with-public-key-aggregation BlsSecretKey *bls.SecretKey + + // StakingFiles contains cert file and key file bytes from which Node ID is derived + StakingFiles *StakingFiles + + DevNetInfo *DevNetInfo } // NewNodeConnection creates a new SSH connection to the node @@ -606,3 +628,39 @@ func (h *Node) HasSystemDAvailable() bool { } return strings.TrimSpace(string(data)) == "systemd" } + +// CreateStakingFiles creates cert, key and bls signer key bytes for a Node +func (h *Node) CreateStakingFiles() error { + certBytes, keyBytes, err := staking.NewCertAndKeyBytes() + if err != nil { + return err + } + + blsSignerKeyBytes, err := utils.NewBlsSecretKeyBytes() + h.StakingFiles = &StakingFiles{ + CertBytes: certBytes, + KeyBytes: keyBytes, + BlsSignerKeyBytes: blsSignerKeyBytes, + } + return nil +} + +func (h *Node) SetDevnetInfo(genesisBytes, nodeConfigBytes []byte) { + h.DevNetInfo = &DevNetInfo{ + GenesisBytes: genesisBytes, + NodeConfigBytes: nodeConfigBytes, + } +} + +func (h *Node) SetNodeID() error { + if h.StakingFiles == nil { + return fmt.Errorf("no staking files are found at node") + } + nodeID, err := utils.ToNodeID(h.StakingFiles.CertBytes) + if err != nil { + return err + } + + h.NodeID = nodeID.String() + return nil +} diff --git a/node/ssh.go b/node/ssh.go index fca99ea..6de6c9b 100644 --- a/node/ssh.go +++ b/node/ssh.go @@ -438,3 +438,43 @@ func (h *Node) RunSSHCopyMonitoringDashboards(monitoringDashboardPath string) er } return nil } + +// RunSSHSetupDevNet runs script to setup devnet +func (h *Node) RunSSHSetupDevNet() error { + if err := h.MkdirAll( + constants.CloudNodeConfigPath, + constants.SSHDirOpsTimeout, + ); err != nil { + return err + } + if err := h.UploadBytes( + h.DevNetInfo.GenesisBytes, + filepath.Join(constants.CloudNodeConfigPath, constants.GenesisFileName), + constants.SSHFileOpsTimeout, + ); err != nil { + return err + } + if err := h.UploadBytes( + h.DevNetInfo.NodeConfigBytes, + filepath.Join(constants.CloudNodeConfigPath, constants.NodeFileName), + constants.SSHFileOpsTimeout, + ); err != nil { + return err + } + if err := h.StopDockerCompose(constants.SSHLongRunningScriptTimeout); err != nil { + return err + } + if err := h.Remove("/home/ubuntu/.avalanchego/db", true); err != nil { + return err + } + if err := h.MkdirAll("/home/ubuntu/.avalanchego/db", constants.SSHDirOpsTimeout); err != nil { + return err + } + if err := h.Remove("/home/ubuntu/.avalanchego/logs", true); err != nil { + return err + } + if err := h.MkdirAll("/home/ubuntu/.avalanchego/logs", constants.SSHDirOpsTimeout); err != nil { + return err + } + return h.StartDockerCompose(constants.SSHLongRunningScriptTimeout) +}