bulk of cosmos

.gitignore

@@ -14,11 +14,14 @@
 # Dependency directories (remove the comment below to include it)
 # vendor/
 
-.env
+.env*
 
 
 .idea/
 .vscode/
 **.DS_Store
+cmd/arkeodrop/__debug_bin
+infra/node-data/
 
-infra/node-data/
+csv/airdrop_GAIA_delegates.csv
+scripts/node_modules/*

Dockerfile

@@ -0,0 +1,21 @@
+FROM golang:1.19-buster as builder
+ARG GITREF=main
+
+WORKDIR /root
+RUN git clone --depth 2 --branch $GITREF https://github.com/arkeonetwork/airdrop.git
+
+WORKDIR /root/airdrop
+RUN make install
+
+# final image
+FROM ubuntu:noble
+
+WORKDIR /usr/local/bin
+COPY --from=builder /go/bin/ .
+
+WORKDIR /root
+
+RUN apt update && apt upgrade -y
+RUN apt install -y redis postgresql-client curl wget jq htop vim ssh
+
+COPY ./infra/scripts ./scripts

Makefile

@@ -1,3 +1,6 @@
+GITREF=main
+IMGTAG=dev
+
 build:
 	go build ./...
 
@@ -11,4 +14,15 @@ db-migrate:
 	tern migrate -c db/tern.conf -m db
 
 install:
-	go install ./cmd/...
+	go install ./cmd/...
+
+docker-build:
+	docker build --no-cache --pull --platform=linux/amd64 --build-arg=GITREF=${GITREF} --rm -f Dockerfile -t airdroputils:${IMGTAG} .
+
+docker-tag:
+	docker tag airdroputils:dev ghcr.io/arkeonetwork/airdroputils:${IMGTAG}
+
+docker-push:
+	docker push ghcr.io/arkeonetwork/airdroputils:${IMGTAG}
+
+push-image: docker-build docker-tag docker-push

cli/cosmos_indexer.go

@@ -0,0 +1,146 @@
+package cli
+
+import (
+	"fmt"
+	"strconv"
+
+	"github.com/ArkeoNetwork/airdrop/indexer"
+	"github.com/ArkeoNetwork/common/utils"
+	"github.com/spf13/cobra"
+)
+
+var (
+	delegationsFromStateExport = &cobra.Command{
+		Use:   "import-delegations [data-directory] [chain] [height]",
+		Short: "import staked delegations from a cosmos daemond export",
+		Run:   runDelegationsFromStateExport,
+		Args:  cobra.ExactValidArgs(3),
+	}
+	liquidityFromStateExport = &cobra.Command{
+		Use:   "import-liquidity [data-directory] [chain] [height]",
+		Short: "import liquidity from a cosmos daemond export",
+		Run:   runLiquidityFromStateExport,
+		Args:  cobra.ExactValidArgs(3),
+	}
+	indexDelegatorsCmd = &cobra.Command{
+		Use:   "delegators [chain]",
+		Short: "gather cosmos-sdk chain data store in our db",
+		Run:   runDelegatorsIndexer,
+		Args:  cobra.ExactValidArgs(1),
+	}
+	indexThorchainLPCmd = &cobra.Command{
+		Use:   "thor-lp [pool]",
+		Short: "gather cosmos-sdk chain liquidity provider data store in our db",
+		Run:   runThorLPIndexer,
+		Args:  cobra.ExactValidArgs(1),
+	}
+)
+
+func runDelegatorsIndexer(cmd *cobra.Command, args []string) {
+	flags := cmd.InheritedFlags()
+	envPath, _ := flags.GetString("env")
+	c := utils.ReadDBConfig(envPath)
+	if c == nil {
+		fmt.Print("error: no config loaded")
+		return
+	}
+
+	chain := args[0]
+	params := indexer.CosmosIndexerParams{Chain: chain, DB: *c}
+	indxr, err := indexer.NewCosmosIndexer(params)
+	if err != nil {
+		cmd.PrintErrf("error creating cosmos indexer: %+v", err)
+		return
+	}
+
+	if err := indxr.IndexCosmosDelegators(); err != nil {
+		fmt.Printf("error indexing delegators: %+v", err)
+		cmd.PrintErrf("error indexing delegators: %+v", err)
+		return
+	}
+}
+
+func runThorSaversIndexer(cmd *cobra.Command, args []string) {
+	// TODO - this is identical to thor-lp with exception of the endpoint
+}
+
+func runThorLPIndexer(cmd *cobra.Command, args []string) {
+	flags := cmd.InheritedFlags()
+	envPath, _ := flags.GetString("env")
+	c := utils.ReadDBConfig(envPath)
+	if c == nil {
+		cmd.PrintErrf("no config for path %s", envPath)
+		return
+	}
+
+	poolName := args[0]
+	params := indexer.CosmosIndexerParams{Chain: "THOR", DB: *c}
+	indxr, err := indexer.NewCosmosIndexer(params)
+	if err != nil {
+		cmd.PrintErrf("error creating cosmos indexer: %+v", err)
+		return
+	}
+
+	if err = indxr.IndexThorLP(poolName); err != nil {
+		cmd.PrintErrf("error indexing LP: %+v", err)
+	}
+}
+
+func runDelegationsFromStateExport(cmd *cobra.Command, args []string) {
+	flags := cmd.InheritedFlags()
+	envPath, _ := flags.GetString("env")
+	c := utils.ReadDBConfig(envPath)
+	if c == nil {
+		cmd.PrintErrf("no config for path %s", envPath)
+		return
+	}
+
+	chain := args[1]
+	baseDataDir := args[0]
+	sheight := args[2]
+	height, err := strconv.ParseInt(sheight, 10, 64)
+	if err != nil {
+		cmd.PrintErrf("error parsing height %s: %+v", sheight, err)
+		return
+	}
+
+	params := indexer.CosmosIndexerParams{Chain: chain, DB: *c}
+	indxr, err := indexer.NewCosmosIndexer(params)
+	if err != nil {
+		cmd.PrintErrf("error creating cosmos indexer: %+v", err)
+		return
+	}
+	if err = indxr.IndexDelegationsFromStateExport(baseDataDir, chain, height); err != nil {
+		cmd.PrintErrf("error indexing Delegations from state export: %+v", err)
+	}
+}
+
+
+func runLiquidityFromStateExport(cmd *cobra.Command, args []string) {
+	flags := cmd.InheritedFlags()
+	envPath, _ := flags.GetString("env")
+	c := utils.ReadDBConfig(envPath)
+	if c == nil {
+		cmd.PrintErrf("no config for path %s", envPath)
+		return
+	}
+
+	chain := args[1]
+	baseDataDir := args[0]
+	sheight := args[2]
+	height, err := strconv.ParseInt(sheight, 10, 64)
+	if err != nil {
+		cmd.PrintErrf("error parsing height %s: %+v", sheight, err)
+		return
+	}
+
+	params := indexer.CosmosIndexerParams{Chain: chain, DB: *c}
+	indxr, err := indexer.NewCosmosIndexer(params)
+	if err != nil {
+		cmd.PrintErrf("error creating cosmos indexer: %+v", err)
+		return
+	}
+	if err = indxr.IndexLiquidityFromStateExport(baseDataDir, chain, height); err != nil {
+		cmd.PrintErrf("error indexing Delegations from state export: %+v", err)
+	}
+}

cli/eth_indexer.go

@@ -0,0 +1,91 @@
+package cli
+
+import (
+	"github.com/ArkeoNetwork/airdrop/indexer"
+	"github.com/ArkeoNetwork/common/logging"
+	"github.com/ArkeoNetwork/common/utils"
+	"github.com/spf13/cobra"
+)
+
+type Config struct {
+	DBHost         string `mapstructure:"DB_HOST"`
+	DBPort         uint   `mapstructure:"DB_PORT"`
+	DBUser         string `mapstructure:"DB_USER"`
+	DBPass         string `mapstructure:"DB_PASS"`
+	DBName         string `mapstructure:"DB_NAME"`
+	DBSSLMode      string `mapstructure:"DB_SSL_MODE"`
+	DBPoolMaxConns int    `mapstructure:"DB_POOL_MAX_CONNS"`
+	DBPoolMinConns int    `mapstructure:"DB_POOL_MIN_CONNS"`
+}
+
+var (
+	log         = logging.WithoutFields()
+	indexEthCmd = &cobra.Command{
+		Use:   "index-eth",
+		Short: "gather eth chain data store in our db",
+		Run:   runEthIndexer,
+	}
+)
+
+func init() {
+	indexEthCmd.Flags().BoolP("transfers", "t", false, "index token transfers")
+	indexEthCmd.Flags().BoolP("rewards", "r", false, "index staking rewards")
+	indexEthCmd.Flags().Bool("hedgeys", false, "index hedgey events")
+}
+
+// index ethereum things
+func runEthIndexer(cmd *cobra.Command, args []string) {
+	log.Info("starting data generation process")
+	flags := cmd.InheritedFlags()
+	envPath, _ := flags.GetString("env")
+	c := utils.ReadDBConfig(envPath)
+	if c == nil {
+		cmd.PrintErr("db config undefined")
+		return
+	}
+
+	indexerApp := indexer.NewIndexer(indexer.IndexerAppParams{
+		DBConfig: *c,
+	})
+	xfers, _ := cmd.Flags().GetBool("transfers")
+	rewards, _ := cmd.Flags().GetBool("rewards")
+	hedgeys, _ := cmd.Flags().GetBool("hedgeys")
+
+	if !xfers && !rewards && !hedgeys {
+		log.Infof("no flags set, starting all eth indexers")
+		indexerApp.Start()
+		return
+	}
+
+	var err error
+	if xfers {
+		if err = indexerApp.IndexTransfers(); err != nil {
+			cmd.PrintErrf("error indexing transfers: %+v", err)
+		}
+	}
+
+	if rewards {
+		if err = indexerApp.IndexStakingRewardsEvents("stakingrewardsv5"); err != nil {
+			cmd.PrintErrf("error indexing staking rewards v5: %+v", err)
+		}
+		if err = indexerApp.IndexStakingRewardsEvents("stakingrewardsv6"); err != nil {
+			cmd.PrintErrf("error indexing staking rewards v6: %+v", err)
+		}
+		if err = indexerApp.IndexStakingRewardsEvents("stakingrewardsv7"); err != nil {
+			cmd.PrintErrf("error indexing staking rewards v7: %+v", err)
+		}
+		if err = indexerApp.IndexStakingRewardsEvents("stakingrewardsv8"); err != nil {
+			cmd.PrintErrf("error indexing staking rewards v8: %+v", err)
+		}
+		if err = indexerApp.IndexStakingRewardsEvents("stakingrewardsv9"); err != nil {
+			cmd.PrintErrf("error indexing staking rewards v9: %+v", err)
+		}
+	}
+
+	if hedgeys {
+		if err = indexerApp.IndexHedgeyEvents(); err != nil {
+			cmd.PrintErrf("error indexing hedgeys: %+v", err)
+		}
+	}
+	log.Debug("finished data generation process")
+}