From 401e5e6fde8c39fcf1eecc597bd96e169ec660be Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Fri, 11 Sep 2020 09:03:08 -0700 Subject: [PATCH 001/376] Update rosetta-sdk-go --- go.mod | 2 +- go.sum | 58 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 59 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index f2db8bac..6eff6a0b 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/coinbase/rosetta-cli go 1.13 require ( - github.com/coinbase/rosetta-sdk-go v0.3.5-0.20200901205702-b739db12905a + github.com/coinbase/rosetta-sdk-go v0.4.1 github.com/fatih/color v1.9.0 github.com/jinzhu/copier v0.0.0-20190924061706-b57f9002281a github.com/olekukonko/tablewriter v0.0.2-0.20190409134802-7e037d187b0c diff --git a/go.sum b/go.sum index c7b73cbb..19c4800d 100644 --- a/go.sum +++ b/go.sum @@ -121,6 +121,8 @@ github.com/coinbase/rosetta-sdk-go v0.3.5-0.20200901204520-f16bdfd018fe h1:wl04M github.com/coinbase/rosetta-sdk-go v0.3.5-0.20200901204520-f16bdfd018fe/go.mod h1:CE6c0Ws+rKwv4yiQDtAOuYwlC3tpXr2Cq5RTE/oVivY= github.com/coinbase/rosetta-sdk-go v0.3.5-0.20200901205702-b739db12905a h1:virCY7xRX4Qi9EvWDordUyvCDlaP3k7ftIMLgcLzs1s= github.com/coinbase/rosetta-sdk-go v0.3.5-0.20200901205702-b739db12905a/go.mod h1:CE6c0Ws+rKwv4yiQDtAOuYwlC3tpXr2Cq5RTE/oVivY= +github.com/coinbase/rosetta-sdk-go v0.4.1 h1:bL8qgwRZ6O56V+drjbdRAhnPZYdkNFjmY5V52v0AaxU= +github.com/coinbase/rosetta-sdk-go v0.4.1/go.mod h1:Luv0AhzZH81eul2hYZ3w0hBGwmFPiexwbntYxihEZck= github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk= @@ -146,6 +148,7 @@ github.com/dgraph-io/badger/v2 v2.0.3 h1:inzdf6VF/NZ+tJ8RwwYMjJMvsOALTHYdozn0qSl github.com/dgraph-io/badger/v2 v2.0.3/go.mod h1:3KY8+bsP8wI0OEnQJAKpd4wIJW/Mm32yw2j/9FUVnIM= github.com/dgraph-io/badger/v2 v2.2007.1 h1:t36VcBCpo4SsmAD5M8wVv1ieVzcALyGfaJ92z4ccULM= github.com/dgraph-io/badger/v2 v2.2007.1/go.mod h1:26P/7fbL4kUZVEVKLAKXkBXKOydDmM2p1e+NhhnBCAE= +github.com/dgraph-io/badger/v2 v2.2007.2/go.mod h1:26P/7fbL4kUZVEVKLAKXkBXKOydDmM2p1e+NhhnBCAE= github.com/dgraph-io/ristretto v0.0.2-0.20200115201040-8f368f2f2ab3/go.mod h1:KPxhHT9ZxKefz+PCeOGsrHpl1qZ7i70dGTu2u+Ahh6E= github.com/dgraph-io/ristretto v0.0.3-0.20200630154024-f66de99634de/go.mod h1:KPxhHT9ZxKefz+PCeOGsrHpl1qZ7i70dGTu2u+Ahh6E= github.com/dgraph-io/ristretto v0.0.3 h1:jh22xisGBjrEVnRZ1DVTpBVQm0Xndu8sMl0CWDzSIBI= @@ -159,6 +162,7 @@ github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8 github.com/dlclark/regexp2 v1.2.0/go.mod h1:2pZnwuY/m+8K6iRw6wQdMtk+rH5tNGR1i55kozfMjCc= github.com/docker/docker v1.4.2-0.20180625184442-8e610b2b55bf/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= github.com/dop251/goja v0.0.0-20200219165308-d1232e640a87/go.mod h1:Mw6PkjjMXWbTj+nnj4s3QPXq1jaT0s5pC0iFD4+BOAA= +github.com/dop251/goja v0.0.0-20200721192441-a695b0cdd498/go.mod h1:Mw6PkjjMXWbTj+nnj4s3QPXq1jaT0s5pC0iFD4+BOAA= github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo= github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/edsrzf/mmap-go v0.0.0-20160512033002-935e0e8a636c/go.mod h1:YO35OhQPt3KJa3ryjFM5Bs14WD66h8eGKpfaBNrHW5M= @@ -168,6 +172,7 @@ github.com/ethereum/go-ethereum v1.9.18 h1:+vzvufVD7+OfQa07IJP20Z7AGZsJaw0M6JIA/ github.com/ethereum/go-ethereum v1.9.18/go.mod h1:JSSTypSMTkGZtAdAChH2wP5dZEvPGh3nUTuDpH+hNrg= github.com/ethereum/go-ethereum v1.9.20 h1:kk/J5OIoaoz3DRrCXznz3RGi212mHHXwzXlY/ZQxcj0= github.com/ethereum/go-ethereum v1.9.20/go.mod h1:JSSTypSMTkGZtAdAChH2wP5dZEvPGh3nUTuDpH+hNrg= +github.com/ethereum/go-ethereum v1.9.21/go.mod h1:RXAVzbGrSGmDkDnHymruTAIEjUR3E4TX0EOpaj702sI= github.com/fatih/color v1.3.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/fatih/color v1.9.0 h1:8xPHl4/q1VyqGIPif1F+1V3Y3lSmrq01EabUW3CoW5s= github.com/fatih/color v1.9.0/go.mod h1:eQcE1qtQxscV5RaZvpXrrb8Drkc3/DdQ+uUYCNjL+zU= @@ -175,6 +180,7 @@ github.com/fatih/structtag v1.2.0 h1:/OdNE99OxoI/PqaW/SuSK9uxxT3f/tcSZgon/ssNSx4 github.com/fatih/structtag v1.2.0/go.mod h1:mBJUNpUnHmRKrKlQQlmCrh5PuhftFbNv8Ys4/aAZl94= github.com/fjl/memsize v0.0.0-20180418122429-ca190fb6ffbc/go.mod h1:VvhXpOYNQvB+uIk2RvXzuaQtkQJzzIx6lSBe1xv7hi0= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= +github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= github.com/gballet/go-libpcsclite v0.0.0-20190607065134-2772fd86a8ff/go.mod h1:x7DCsMOv1taUwEWCzT4cmDeAkigA5/QCwUodaVOe8Ww= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= @@ -209,6 +215,7 @@ github.com/golang/snappy v0.0.2-0.20200707131729-196ae77b8a26 h1:lMm2hD9Fy0ynom5 github.com/golang/snappy v0.0.2-0.20200707131729-196ae77b8a26/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/google/addlicense v0.0.0-20200817051935-6f4cd4aacc89 h1:oTppmscIAQ2Y1tcsMDcTLR3z4MN/96/pvIsBSLGl7o8= github.com/google/addlicense v0.0.0-20200817051935-6f4cd4aacc89/go.mod h1:EMjYTRimagHs1FwlIqKyX3wAM0u3rA+McvlIIWmSamA= +github.com/google/addlicense v0.0.0-20200827091314-d1655b921368/go.mod h1:EMjYTRimagHs1FwlIqKyX3wAM0u3rA+McvlIIWmSamA= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= @@ -217,7 +224,10 @@ github.com/google/go-cmp v0.4.0 h1:xsAVV57WRhGj6kEIi8ReJzQlHHqcBYCElAvkovg3B/4= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.0 h1:/QaMHBdZ26BB3SSst0Iwl10Epc+xhTquomWX0oZEB6w= github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/pprof v0.0.0-20181127221834-b4f47329b966/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= +github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/gorilla/mux v1.7.4/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= @@ -290,6 +300,7 @@ github.com/mitchellh/mapstructure v1.3.3/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RR github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/naoina/go-stringutil v0.1.0/go.mod h1:XJ2SJL9jCtBh+P9q5btrd/Ylo8XwT/h1USek5+NqSA0= github.com/naoina/toml v0.1.2-0.20170918210437-9fafd6967416/go.mod h1:NBIhNtsFMo3G2szEBne+bO4gS192HuIYRqfvOWb4i1E= +github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= github.com/olekukonko/tablewriter v0.0.1/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo= github.com/olekukonko/tablewriter v0.0.2-0.20190409134802-7e037d187b0c h1:1RHs3tNxjXGHeul8z2t6H2N2TlAqpKe5yryJztRx4Jk= @@ -297,9 +308,13 @@ github.com/olekukonko/tablewriter v0.0.2-0.20190409134802-7e037d187b0c/go.mod h1 github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.10.2/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= +github.com/onsi/ginkgo v1.14.0/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY= github.com/onsi/gomega v1.4.1/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= +github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= +github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= github.com/pborman/uuid v0.0.0-20170112150404-1b00554d8222/go.mod h1:VyrYX9gd7irzKovcSS6BIIEwPRkP2Wm2m9ufcdFSJ34= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= @@ -324,12 +339,18 @@ github.com/prometheus/tsdb v0.6.2-0.20190402121629-4f204dcbc150/go.mod h1:qhTCs0 github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= github.com/rjeczalik/notify v0.9.1/go.mod h1:rKwnCoCGeuQnwBtTSPL9Dad03Vh2n40ePRrjvIXnJho= github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= +github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= +github.com/rogpeppe/go-internal v1.5.2/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= +github.com/rogpeppe/go-internal v1.6.0/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= +github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= github.com/rs/cors v0.0.0-20160617231935-a62a804a8a00/go.mod h1:gFx+x8UowdsKA9AchylcLynDq+nNFfI8FkUZdN/jGCU= github.com/rs/xhandler v0.0.0-20160618193221-ed27b6fd6521/go.mod h1:RvLn4FgxWubrpZHtQLnOf6EwhN2hEMusxZOhcW9H3UQ= github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/segmentio/golines v0.0.0-20200808004416-0a9796b248e8/go.mod h1:bQSh5qdVR67XiCKbaVvYO41s50c5hQo+3cY/1CQQ3xQ= +github.com/segmentio/golines v0.0.0-20200824192126-7f30d3046793/go.mod h1:bQSh5qdVR67XiCKbaVvYO41s50c5hQo+3cY/1CQQ3xQ= github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= +github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= github.com/shirou/gopsutil v2.20.5+incompatible h1:tYH07UPoQt0OCQdgWWMgYHy3/a9bcxNpBIysykNIP7I= github.com/shirou/gopsutil v2.20.5+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA= github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= @@ -370,6 +391,7 @@ github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5 github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/syndtr/goleveldb v1.0.1-0.20190923125748-758128399b1d/go.mod h1:9OrXJhf154huy1nPWmuSrkgjPUtUNhA+Zmy+6AESzuA= +github.com/syndtr/goleveldb v1.0.1-0.20200815110645-5c35d600f0ca/go.mod h1:u2MKkTVTVJWe5D1rCvame8WqhBd88EuIwODJZ1VHCPM= github.com/tidwall/gjson v1.6.0 h1:9VEQWz6LLMUsUl6PueE49ir4Ka6CzLymOAZDxpFsTDc= github.com/tidwall/gjson v1.6.0/go.mod h1:P256ACg0Mn+j1RXIDXoss50DeIABTYK1PULOJHhxOls= github.com/tidwall/gjson v1.6.1 h1:LRbvNuNuvAiISWg6gxLEFuCe72UKy5hDqhxW/8183ws= @@ -398,6 +420,9 @@ github.com/x-cray/logrus-prefixed-formatter v0.5.2 h1:00txxvfBM9muc0jiLIEAkAcIMJ github.com/x-cray/logrus-prefixed-formatter v0.5.2/go.mod h1:2duySbKsL6M18s5GU7VPsoEPHyzalCE06qoARUCeBBE= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= +github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= @@ -408,6 +433,7 @@ golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnf golang.org/x/crypto v0.0.0-20181127143415-eb0de9b17e85/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550 h1:ObdrDkeb4kJdCP557AjRjq69pTHfNouLtWZG7j9rPN8= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200115085410-6d4e4cb37c7d/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= @@ -416,11 +442,15 @@ golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 h1:psW17arqaxU48Z5kZ0CQnk golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200728195943-123391ffb6de h1:ikNHVSjEfnvz6sxdSPCaPt572qowuyMDMJLLm3Db3ig= golang.org/x/crypto v0.0.0-20200728195943-123391ffb6de/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20200820211705-5c72a883971a/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3 h1:XQyxROzUlZH+WIQwySDgnISgOivlhjIEwaQaJEJrrN0= golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= +golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/net v0.0.0-20180719180050-a680a1efc54d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -435,11 +465,15 @@ golang.org/x/net v0.0.0-20190522155817-f3200d17e092/go.mod h1:HSz+uSET+XFnRR8LxR golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= golang.org/x/net v0.0.0-20190620200207-3b0461eec859 h1:R/3boaszxrf1GEUWTVDzSKVwLmSJpwZ1yqXm8j0v2QI= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200625001655-4c5254603344 h1:vGXIOMxbNfDTk/aXCmfdLgkrSV+Z2tcbze+pEc3v5W4= golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20200813134508-3edf25e44fcc h1:zK/HqS5bZxDptfPJNq8v7vJfXtkU7r9TLIoSr1bXaP4= golang.org/x/net v0.0.0-20200813134508-3edf25e44fcc/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20200904194848-62affa334b73/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -461,17 +495,25 @@ golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb h1:fgwFCsaw9buMuxNd6+DQfAuSFqbNiQZpcgJQAgJsK6k= golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191008105621-543471e840be/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191024172528-b4ff53e7a1cb h1:ZxSglHghKPYD8WDeRUzRJrUJtDF0PxsTUSxyqr9/5BI= golang.org/x/sys v0.0.0-20191024172528-b4ff53e7a1cb/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200519105757-fe76b779f299/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200814200057-3d37ad5750ed/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200817155316-9781c653f443 h1:X18bCaipMcoJGm27Nv7zr4XYPKGUy92GtqboKC2Hxaw= golang.org/x/sys v0.0.0-20200817155316-9781c653f443/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200824131525-c12d262b63d8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200909081042-eff7692f9009/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -482,7 +524,15 @@ golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3 golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20191024220359-3d91e92cde03 h1:4gtJXHJ9ud0q8MNSDxJsRU/WH+afypbe4Vk4zq+8qow= golang.org/x/tools v0.0.0-20191024220359-3d91e92cde03/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191130070609-6e064ea0cf2d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20200626171337-aa94e735be7f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200729181040-64cdafbe085c/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20200731060945-b5fad4ed8dd6/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20200904185747-39188db58858/go.mod h1:Cj7w3i3Rnn0Xh82ur9kSqwfTHTeVxaDqrfMjpcNT6bE= +golang.org/x/tools/gopls v0.4.4/go.mod h1:zhyGzA+CAtREUwwq/btQxEx2FHnGzDwJvGs5YqdVCbE= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= @@ -517,6 +567,7 @@ gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/natefinch/npipe.v2 v2.0.0-20160621034901-c1b8fa8bdcce/go.mod h1:5AcXVHNjg+BDxry382+8OKon8SEWiKktQR07RKPsv1c= gopkg.in/olebedev/go-duktape.v3 v3.0.0-20200619000410-60c24ae608a6/go.mod h1:uAJfkITjFhyEEuUfm7bsmCZRbW5WRq8s9EY8HZ6hCns= @@ -528,10 +579,17 @@ gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bl gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= +honnef.co/go/tools v0.0.1-2020.1.5/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= +mvdan.cc/gofumpt v0.0.0-20200709182408-4fd085cb6d5f/go.mod h1:9VQ397fNXEnF84t90W4r4TRCQK+pg9f8ugVfyj+S26w= +mvdan.cc/gofumpt v0.0.0-20200802201014-ab5a8192947d/go.mod h1:bzrjFmaD6+xqohD3KYP0H2FEuxknnBmyyOxdhLdaIws= +mvdan.cc/xurls/v2 v2.2.0/go.mod h1:EV1RMtya9D6G5DMYPGD8zTQzaHet6Jh8gFlRgGRJeO8= src.techknowlogick.com/xgo v1.1.1-0.20200814033943-12cf2e8194ca h1:kumZMjGGyzQDXLgV1CywLHdhnLtaFxDui/2IXqPtVMw= src.techknowlogick.com/xgo v1.1.1-0.20200814033943-12cf2e8194ca/go.mod h1:31CE1YKtDOrKTk9PSnjTpe6YbO6W/0LTYZ1VskL09oU= From bf88009115c23a9df6f93bf71766d9c3e5c948d9 Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Fri, 11 Sep 2020 09:20:11 -0700 Subject: [PATCH 002/376] Update configuration --- configuration/configuration.go | 64 +++++++++++++++++------------ configuration/configuration_test.go | 32 ++++++++------- go.sum | 4 ++ 3 files changed, 58 insertions(+), 42 deletions(-) diff --git a/configuration/configuration.go b/configuration/configuration.go index 997c3bf8..fc9e826d 100644 --- a/configuration/configuration.go +++ b/configuration/configuration.go @@ -53,13 +53,14 @@ const ( // Default Configuration Values const ( DefaultURL = "http://localhost:8080" - DefaultSyncConcurrency = 8 - DefaultTransactionConcurrency = 16 + DefaultTimeout = 10 + DefaultRetryElapsedTime = 60 + DefaultMaxOnlineConnections = 120 // most OS have a default limit of 128 + DefaultMaxOfflineConnections = 4 // we shouldn't need many connections for construction + DefaultMaxSyncConcurrency = 64 DefaultActiveReconciliationConcurrency = 16 DefaultInactiveReconciliationConcurrency = 4 DefaultInactiveReconciliationFrequency = 250 - DefaultTimeout = 10 - DefaultRetryElapsedTime = 60 DefaultConfirmationDepth = 10 DefaultStaleDepth = 30 DefaultBroadcastLimit = 3 @@ -85,6 +86,10 @@ type ConstructionConfiguration struct { // OfflineURL is the URL of a Rosetta API implementation in "offline mode". OfflineURL string `json:"offline_url"` + // MaxOffineConnections is the maximum number of open connections that the offline + // fetcher will open. + MaxOfflineConnections uint64 `json:"max_offline_connections"` + // StaleDepth is the number of blocks to wait before attempting // to rebroadcast after not finding a transaction on-chain. StaleDepth int64 `json:"stale_depth"` @@ -150,14 +155,14 @@ func DefaultDataConfiguration() *DataConfiguration { // DefaultConstructionConfiguration and DefaultDataConfiguration. func DefaultConfiguration() *Configuration { return &Configuration{ - Network: EthereumNetwork, - OnlineURL: DefaultURL, - HTTPTimeout: DefaultTimeout, - RetryElapsedTime: DefaultRetryElapsedTime, - SyncConcurrency: DefaultSyncConcurrency, - TransactionConcurrency: DefaultTransactionConcurrency, - TipDelay: DefaultTipDelay, - Data: DefaultDataConfiguration(), + Network: EthereumNetwork, + OnlineURL: DefaultURL, + MaxOnlineConnections: DefaultMaxOnlineConnections, + HTTPTimeout: DefaultTimeout, + RetryElapsedTime: DefaultRetryElapsedTime, + MaxSyncConcurrency: DefaultMaxSyncConcurrency, + TipDelay: DefaultTipDelay, + Data: DefaultDataConfiguration(), } } @@ -291,11 +296,13 @@ type Configuration struct { // RetryElapsedTime is the total time to spend retrying a HTTP request in seconds. RetryElapsedTime uint64 `json:"retry_elapsed_time"` - // SyncConcurrency is the concurrency to use while syncing blocks. - SyncConcurrency uint64 `json:"sync_concurrency"` + // MaxOnlineConnections is the maximum number of open connections that the online + // fetcher will open. + MaxOnlineConnections uint64 `json:"max_online_connections"` - // TransactionConcurrency is the concurrency to use while fetching transactions (if required). - TransactionConcurrency uint64 `json:"transaction_concurrency"` + // MaxSyncConcurrency is the maximum sync concurrency to use while syncing blocks. + // Sync concurrency is managed automatically by the `syncer` package. + MaxSyncConcurrency uint64 `json:"max_sync_concurrency"` // TipDelay dictates how many seconds behind the current time is considered // tip. If we are > TipDelay seconds from the last processed block, @@ -325,6 +332,10 @@ func populateConstructionMissingFields( constructionConfig.OfflineURL = DefaultURL } + if constructionConfig.MaxOfflineConnections == 0 { + constructionConfig.MaxOfflineConnections = DefaultMaxOfflineConnections + } + if constructionConfig.StaleDepth == 0 { constructionConfig.StaleDepth = DefaultStaleDepth } @@ -381,12 +392,12 @@ func populateMissingFields(config *Configuration) *Configuration { config.RetryElapsedTime = DefaultRetryElapsedTime } - if config.SyncConcurrency == 0 { - config.SyncConcurrency = DefaultSyncConcurrency + if config.MaxOnlineConnections == 0 { + config.MaxOnlineConnections = DefaultMaxOnlineConnections } - if config.TransactionConcurrency == 0 { - config.TransactionConcurrency = DefaultTransactionConcurrency + if config.MaxSyncConcurrency == 0 { + config.MaxSyncConcurrency = DefaultMaxSyncConcurrency } if config.TipDelay == 0 { @@ -442,19 +453,18 @@ func assertConstructionConfiguration(config *ConstructionConfiguration) error { ) } - // Checks if valid curvetype - err = asserter.CurveType(account.CurveType) - if err != nil { + // Checks if valid CurveType + if err := asserter.CurveType(account.CurveType); err != nil { return fmt.Errorf("%w: invalid CurveType for prefunded account", err) } - // Checks if address is not empty string - if account.Address == "" { + // Checks if valid AccountIdentifier + if err := asserter.AccountIdentifier(account.AccountIdentifier); err != nil { return fmt.Errorf("Account.Address is missing for prefunded account") } - // Check if currency is valid - err = asserter.Amount(&types.Amount{Value: "0", Currency: account.Currency}) + // Check if valid Currency + err = asserter.Currency(account.Currency) if err != nil { return fmt.Errorf("%w: invalid currency for prefunded account", err) } diff --git a/configuration/configuration_test.go b/configuration/configuration_test.go index 5ec55bf6..0fae2208 100644 --- a/configuration/configuration_test.go +++ b/configuration/configuration_test.go @@ -47,17 +47,18 @@ var ( Blockchain: "sweet", Network: "sweeter", }, - OnlineURL: "http://hasudhasjkdk", - HTTPTimeout: 21, - RetryElapsedTime: 1000, - SyncConcurrency: 12, - TransactionConcurrency: 2, - TipDelay: 1231, + OnlineURL: "http://hasudhasjkdk", + MaxOnlineConnections: 10, + HTTPTimeout: 21, + RetryElapsedTime: 1000, + MaxSyncConcurrency: 12, + TipDelay: 1231, Construction: &ConstructionConfiguration{ - OfflineURL: "https://ashdjaksdkjshdk", - StaleDepth: 12, - BroadcastLimit: 200, - BlockBroadcastLimit: 992, + OfflineURL: "https://ashdjaksdkjshdk", + MaxOfflineConnections: 21, + StaleDepth: 12, + BroadcastLimit: 200, + BlockBroadcastLimit: 992, Workflows: append( fakeWorkflows, &job.Workflow{ @@ -146,11 +147,12 @@ func TestLoadConfiguration(t *testing.T) { expected: func() *Configuration { cfg := DefaultConfiguration() cfg.Construction = &ConstructionConfiguration{ - OfflineURL: DefaultURL, - StaleDepth: DefaultStaleDepth, - BroadcastLimit: DefaultBroadcastLimit, - BlockBroadcastLimit: DefaultBlockBroadcastLimit, - Workflows: fakeWorkflows, + OfflineURL: DefaultURL, + MaxOfflineConnections: DefaultMaxOfflineConnections, + StaleDepth: DefaultStaleDepth, + BroadcastLimit: DefaultBroadcastLimit, + BlockBroadcastLimit: DefaultBlockBroadcastLimit, + Workflows: fakeWorkflows, } return cfg diff --git a/go.sum b/go.sum index 19c4800d..057f4417 100644 --- a/go.sum +++ b/go.sum @@ -148,6 +148,7 @@ github.com/dgraph-io/badger/v2 v2.0.3 h1:inzdf6VF/NZ+tJ8RwwYMjJMvsOALTHYdozn0qSl github.com/dgraph-io/badger/v2 v2.0.3/go.mod h1:3KY8+bsP8wI0OEnQJAKpd4wIJW/Mm32yw2j/9FUVnIM= github.com/dgraph-io/badger/v2 v2.2007.1 h1:t36VcBCpo4SsmAD5M8wVv1ieVzcALyGfaJ92z4ccULM= github.com/dgraph-io/badger/v2 v2.2007.1/go.mod h1:26P/7fbL4kUZVEVKLAKXkBXKOydDmM2p1e+NhhnBCAE= +github.com/dgraph-io/badger/v2 v2.2007.2 h1:EjjK0KqwaFMlPin1ajhP943VPENHJdEz1KLIegjaI3k= github.com/dgraph-io/badger/v2 v2.2007.2/go.mod h1:26P/7fbL4kUZVEVKLAKXkBXKOydDmM2p1e+NhhnBCAE= github.com/dgraph-io/ristretto v0.0.2-0.20200115201040-8f368f2f2ab3/go.mod h1:KPxhHT9ZxKefz+PCeOGsrHpl1qZ7i70dGTu2u+Ahh6E= github.com/dgraph-io/ristretto v0.0.3-0.20200630154024-f66de99634de/go.mod h1:KPxhHT9ZxKefz+PCeOGsrHpl1qZ7i70dGTu2u+Ahh6E= @@ -172,6 +173,7 @@ github.com/ethereum/go-ethereum v1.9.18 h1:+vzvufVD7+OfQa07IJP20Z7AGZsJaw0M6JIA/ github.com/ethereum/go-ethereum v1.9.18/go.mod h1:JSSTypSMTkGZtAdAChH2wP5dZEvPGh3nUTuDpH+hNrg= github.com/ethereum/go-ethereum v1.9.20 h1:kk/J5OIoaoz3DRrCXznz3RGi212mHHXwzXlY/ZQxcj0= github.com/ethereum/go-ethereum v1.9.20/go.mod h1:JSSTypSMTkGZtAdAChH2wP5dZEvPGh3nUTuDpH+hNrg= +github.com/ethereum/go-ethereum v1.9.21 h1:8qRlhzrItnmUGdVlBzZLI2Tb46S0RdSNjFwICo781ws= github.com/ethereum/go-ethereum v1.9.21/go.mod h1:RXAVzbGrSGmDkDnHymruTAIEjUR3E4TX0EOpaj702sI= github.com/fatih/color v1.3.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/fatih/color v1.9.0 h1:8xPHl4/q1VyqGIPif1F+1V3Y3lSmrq01EabUW3CoW5s= @@ -473,6 +475,7 @@ golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81R golang.org/x/net v0.0.0-20200813134508-3edf25e44fcc h1:zK/HqS5bZxDptfPJNq8v7vJfXtkU7r9TLIoSr1bXaP4= golang.org/x/net v0.0.0-20200813134508-3edf25e44fcc/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20200904194848-62affa334b73 h1:MXfv8rhZWmFeqX3GNZRsd6vOLoaCHjYEX3qkRo3YBUA= golang.org/x/net v0.0.0-20200904194848-62affa334b73/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -510,6 +513,7 @@ golang.org/x/sys v0.0.0-20200814200057-3d37ad5750ed/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200817155316-9781c653f443 h1:X18bCaipMcoJGm27Nv7zr4XYPKGUy92GtqboKC2Hxaw= golang.org/x/sys v0.0.0-20200817155316-9781c653f443/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200824131525-c12d262b63d8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200909081042-eff7692f9009 h1:W0lCpv29Hv0UaM1LXb9QlBHLNP8UFfcKjblhVCWftOM= golang.org/x/sys v0.0.0-20200909081042-eff7692f9009/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= From 8cbafbb23135d8108e1354d5320390787a773bac Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Fri, 11 Sep 2020 09:26:10 -0700 Subject: [PATCH 003/376] Update rosetta-sdk-go --- go.mod | 2 +- go.sum | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index 6eff6a0b..14db5cb9 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/coinbase/rosetta-cli go 1.13 require ( - github.com/coinbase/rosetta-sdk-go v0.4.1 + github.com/coinbase/rosetta-sdk-go v0.4.2-0.20200911162523-bbfbd83145d4 github.com/fatih/color v1.9.0 github.com/jinzhu/copier v0.0.0-20190924061706-b57f9002281a github.com/olekukonko/tablewriter v0.0.2-0.20190409134802-7e037d187b0c diff --git a/go.sum b/go.sum index 057f4417..c468c1f6 100644 --- a/go.sum +++ b/go.sum @@ -123,6 +123,8 @@ github.com/coinbase/rosetta-sdk-go v0.3.5-0.20200901205702-b739db12905a h1:virCY github.com/coinbase/rosetta-sdk-go v0.3.5-0.20200901205702-b739db12905a/go.mod h1:CE6c0Ws+rKwv4yiQDtAOuYwlC3tpXr2Cq5RTE/oVivY= github.com/coinbase/rosetta-sdk-go v0.4.1 h1:bL8qgwRZ6O56V+drjbdRAhnPZYdkNFjmY5V52v0AaxU= github.com/coinbase/rosetta-sdk-go v0.4.1/go.mod h1:Luv0AhzZH81eul2hYZ3w0hBGwmFPiexwbntYxihEZck= +github.com/coinbase/rosetta-sdk-go v0.4.2-0.20200911162523-bbfbd83145d4 h1:aEQB1Y2qp036Cfy+Cm2pjtPEKmFofw/QKzfd3JL/wIU= +github.com/coinbase/rosetta-sdk-go v0.4.2-0.20200911162523-bbfbd83145d4/go.mod h1:Luv0AhzZH81eul2hYZ3w0hBGwmFPiexwbntYxihEZck= github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk= From 10971ba1c1f10ddd2f88961f369b8838b7f546bd Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Fri, 11 Sep 2020 09:50:14 -0700 Subject: [PATCH 004/376] Update scripts to use a particular go version on compilation --- scripts/compile.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/compile.sh b/scripts/compile.sh index 77fc9255..151edf2e 100755 --- a/scripts/compile.sh +++ b/scripts/compile.sh @@ -16,7 +16,7 @@ VERSION=$1; -xgo --targets=darwin/*,windows/*,linux/* -out "bin/rosetta-cli-${VERSION}" .; +xgo -go go-1.15.2 --targets=darwin/*,windows/*,linux/* -out "bin/rosetta-cli-${VERSION}" .; # Rename some files mv "bin/rosetta-cli-${VERSION}-darwin-10.6-amd64" "bin/rosetta-cli-${VERSION}-darwin-amd64" From 5be6302428cbb91c7a354229f911a3cca53f35b9 Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Fri, 11 Sep 2020 10:12:47 -0700 Subject: [PATCH 005/376] Fix coordinator helper --- cmd/check_construction.go | 2 +- cmd/check_data.go | 2 +- configuration/configuration.go | 6 +-- pkg/processor/coordinator_helper.go | 62 ++++++++++++++++++----------- pkg/tester/construction.go | 3 +- pkg/tester/data.go | 2 +- 6 files changed, 46 insertions(+), 31 deletions(-) diff --git a/cmd/check_construction.go b/cmd/check_construction.go index 470ebad0..01378c57 100644 --- a/cmd/check_construction.go +++ b/cmd/check_construction.go @@ -62,7 +62,7 @@ func runCheckConstructionCmd(cmd *cobra.Command, args []string) { fetcher := fetcher.New( Config.OnlineURL, - fetcher.WithTransactionConcurrency(Config.TransactionConcurrency), + fetcher.WithMaxConnections(Config.MaxOnlineConnections), fetcher.WithRetryElapsedTime(time.Duration(Config.RetryElapsedTime)*time.Second), fetcher.WithTimeout(time.Duration(Config.HTTPTimeout)*time.Second), ) diff --git a/cmd/check_data.go b/cmd/check_data.go index 2e1c13c7..ac2e42d8 100644 --- a/cmd/check_data.go +++ b/cmd/check_data.go @@ -75,7 +75,7 @@ func runCheckDataCmd(cmd *cobra.Command, args []string) { fetcher := fetcher.New( Config.OnlineURL, - fetcher.WithTransactionConcurrency(Config.TransactionConcurrency), + fetcher.WithMaxConnections(Config.MaxOnlineConnections), fetcher.WithRetryElapsedTime(time.Duration(Config.RetryElapsedTime)*time.Second), fetcher.WithTimeout(time.Duration(Config.HTTPTimeout)*time.Second), ) diff --git a/configuration/configuration.go b/configuration/configuration.go index fc9e826d..9a63676e 100644 --- a/configuration/configuration.go +++ b/configuration/configuration.go @@ -88,7 +88,7 @@ type ConstructionConfiguration struct { // MaxOffineConnections is the maximum number of open connections that the offline // fetcher will open. - MaxOfflineConnections uint64 `json:"max_offline_connections"` + MaxOfflineConnections int `json:"max_offline_connections"` // StaleDepth is the number of blocks to wait before attempting // to rebroadcast after not finding a transaction on-chain. @@ -298,11 +298,11 @@ type Configuration struct { // MaxOnlineConnections is the maximum number of open connections that the online // fetcher will open. - MaxOnlineConnections uint64 `json:"max_online_connections"` + MaxOnlineConnections int `json:"max_online_connections"` // MaxSyncConcurrency is the maximum sync concurrency to use while syncing blocks. // Sync concurrency is managed automatically by the `syncer` package. - MaxSyncConcurrency uint64 `json:"max_sync_concurrency"` + MaxSyncConcurrency int64 `json:"max_sync_concurrency"` // TipDelay dictates how many seconds behind the current time is considered // tip. If we are > TipDelay seconds from the last processed block, diff --git a/pkg/processor/coordinator_helper.go b/pkg/processor/coordinator_helper.go index 8edd7bb7..5935d90d 100644 --- a/pkg/processor/coordinator_helper.go +++ b/pkg/processor/coordinator_helper.go @@ -83,18 +83,18 @@ func (c *CoordinatorHelper) Derive( networkIdentifier *types.NetworkIdentifier, publicKey *types.PublicKey, metadata map[string]interface{}, -) (string, map[string]interface{}, error) { - add, metadata, fetchErr := c.offlineFetcher.ConstructionDerive( +) (*types.AccountIdentifier, map[string]interface{}, error) { + account, metadata, fetchErr := c.offlineFetcher.ConstructionDerive( ctx, networkIdentifier, publicKey, metadata, ) if fetchErr != nil { - return "", nil, fetchErr.Err + return nil, nil, fetchErr.Err } - return add, metadata, nil + return account, metadata, nil } // Preprocess calls the /construction/preprocess endpoint @@ -104,8 +104,8 @@ func (c *CoordinatorHelper) Preprocess( networkIdentifier *types.NetworkIdentifier, intent []*types.Operation, metadata map[string]interface{}, -) (map[string]interface{}, error) { - res, fetchErr := c.offlineFetcher.ConstructionPreprocess( +) (map[string]interface{}, []*types.AccountIdentifier, error) { + options, requiredPublicKeys, fetchErr := c.offlineFetcher.ConstructionPreprocess( ctx, networkIdentifier, intent, @@ -113,10 +113,10 @@ func (c *CoordinatorHelper) Preprocess( ) if fetchErr != nil { - return nil, fetchErr.Err + return nil, nil, fetchErr.Err } - return res, nil + return options, requiredPublicKeys, nil } // Metadata calls the /construction/metadata endpoint @@ -125,18 +125,20 @@ func (c *CoordinatorHelper) Metadata( ctx context.Context, networkIdentifier *types.NetworkIdentifier, metadataRequest map[string]interface{}, -) (map[string]interface{}, error) { - res, fetchErr := c.offlineFetcher.ConstructionMetadata( + publicKeys []*types.PublicKey, +) (map[string]interface{}, []*types.Amount, error) { + metadata, suggestedFee, fetchErr := c.offlineFetcher.ConstructionMetadata( ctx, networkIdentifier, metadataRequest, + publicKeys, ) if fetchErr != nil { - return nil, fetchErr.Err + return nil, nil, fetchErr.Err } - return res, nil + return metadata, suggestedFee, nil } // Payloads calls the /construction/payloads endpoint @@ -146,12 +148,14 @@ func (c *CoordinatorHelper) Payloads( networkIdentifier *types.NetworkIdentifier, intent []*types.Operation, requiredMetadata map[string]interface{}, + publicKeys []*types.PublicKey, ) (string, []*types.SigningPayload, error) { res, payloads, fetchErr := c.offlineFetcher.ConstructionPayloads( ctx, networkIdentifier, intent, requiredMetadata, + publicKeys, ) if fetchErr != nil { @@ -168,7 +172,7 @@ func (c *CoordinatorHelper) Parse( networkIdentifier *types.NetworkIdentifier, signed bool, transaction string, -) ([]*types.Operation, []string, map[string]interface{}, error) { +) ([]*types.Operation, []*types.AccountIdentifier, map[string]interface{}, error) { ops, signers, metadata, fetchErr := c.offlineFetcher.ConstructionParse( ctx, networkIdentifier, @@ -234,19 +238,29 @@ func (c *CoordinatorHelper) Sign( return c.keyStorage.Sign(ctx, payloads) } +// GetKey is called to get the *types.KeyPair +// associated with an address. +func (c *CoordinatorHelper) GetKey( + ctx context.Context, + dbTx storage.DatabaseTransaction, + account *types.AccountIdentifier, +) (*keys.KeyPair, error) { + return c.keyStorage.GetTransactional(ctx, dbTx, account) +} + // StoreKey stores a KeyPair and address // in KeyStorage. func (c *CoordinatorHelper) StoreKey( ctx context.Context, dbTx storage.DatabaseTransaction, - address string, + account *types.AccountIdentifier, keyPair *keys.KeyPair, ) error { // We optimisically add the interesting address although the dbTx could be reverted. - c.balanceStorageHelper.AddInterestingAddress(address) + c.balanceStorageHelper.AddInterestingAddress(account.Address) _, _ = c.counterStorage.UpdateTransactional(ctx, dbTx, storage.AddressesCreatedCounter, big.NewInt(1)) - return c.keyStorage.StoreTransactional(ctx, address, keyPair, dbTx) + return c.keyStorage.StoreTransactional(ctx, account, keyPair, dbTx) } // Balance returns the balance @@ -299,13 +313,13 @@ func (c *CoordinatorHelper) Coins( return coinsToReturn, nil } -// LockedAddresses returns a slice of all addresses currently sending or receiving +// LockedAccounts returns a slice of all accounts currently sending or receiving // funds. -func (c *CoordinatorHelper) LockedAddresses( +func (c *CoordinatorHelper) LockedAccounts( ctx context.Context, dbTx storage.DatabaseTransaction, -) ([]string, error) { - return c.broadcastStorage.LockedAddresses(ctx, dbTx) +) ([]*types.AccountIdentifier, error) { + return c.broadcastStorage.LockedAccounts(ctx, dbTx) } // AllBroadcasts returns a slice of all in-progress broadcasts in BroadcastStorage. @@ -348,12 +362,12 @@ func (c *CoordinatorHelper) BroadcastAll( return c.broadcastStorage.BroadcastAll(ctx, true) } -// AllAddresses returns a slice of all known addresses. -func (c *CoordinatorHelper) AllAddresses( +// AllAccounts returns a slice of all known accounts. +func (c *CoordinatorHelper) AllAccounts( ctx context.Context, dbTx storage.DatabaseTransaction, -) ([]string, error) { - return c.keyStorage.GetAllAddressesTransactional(ctx, dbTx) +) ([]*types.AccountIdentifier, error) { + return c.keyStorage.GetAllAccountsTransactional(ctx, dbTx) } // HeadBlockExists returns a boolean indicating if a block has been diff --git a/pkg/tester/construction.go b/pkg/tester/construction.go index 850d8881..1bea5711 100644 --- a/pkg/tester/construction.go +++ b/pkg/tester/construction.go @@ -136,6 +136,7 @@ func InitializeConstruction( ) offlineFetcher := fetcher.New( config.Construction.OfflineURL, + fetcher.WithMaxConnections(config.Construction.MaxOfflineConnections), fetcher.WithAsserter(onlineFetcher.Asserter), fetcher.WithTimeout(time.Duration(config.HTTPTimeout)*time.Second), ) @@ -235,7 +236,7 @@ func InitializeConstruction( logger, cancel, []storage.BlockWorker{balanceStorage, coinStorage, broadcastStorage}, - config.SyncConcurrency, + config.MaxSyncConcurrency, ) return &ConstructionTester{ diff --git a/pkg/tester/data.go b/pkg/tester/data.go index 36f24be9..97bd2ad1 100644 --- a/pkg/tester/data.go +++ b/pkg/tester/data.go @@ -263,7 +263,7 @@ func InitializeData( logger, cancel, blockWorkers, - config.SyncConcurrency, + config.MaxSyncConcurrency, ) return &DataTester{ From 222c8f2080a2d232f822bb34f04f6a9cd8af8fe0 Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Fri, 11 Sep 2020 10:27:01 -0700 Subject: [PATCH 006/376] Update check data results --- pkg/tester/construction.go | 17 ++++++++--------- pkg/tester/data.go | 5 ++++- pkg/tester/data_results.go | 15 ++++----------- pkg/tester/data_results_test.go | 3 ++- 4 files changed, 18 insertions(+), 22 deletions(-) diff --git a/pkg/tester/construction.go b/pkg/tester/construction.go index 1bea5711..e6ef0f44 100644 --- a/pkg/tester/construction.go +++ b/pkg/tester/construction.go @@ -31,6 +31,7 @@ import ( "github.com/coinbase/rosetta-sdk-go/parser" "github.com/coinbase/rosetta-sdk-go/statefulsyncer" "github.com/coinbase/rosetta-sdk-go/storage" + "github.com/coinbase/rosetta-sdk-go/syncer" "github.com/coinbase/rosetta-sdk-go/types" "github.com/coinbase/rosetta-sdk-go/utils" "github.com/fatih/color" @@ -148,26 +149,23 @@ func InitializeConstruction( } // Load all accounts for network - addresses, err := keyStorage.GetAllAddresses(ctx) + accounts, err := keyStorage.GetAllAccounts(ctx) if err != nil { return nil, fmt.Errorf("%w: unable to load addresses", err) } - log.Printf("construction tester initialized with %d addresses\n", len(addresses)) - // Track balances on all addresses - for _, address := range addresses { - balanceStorageHelper.AddInterestingAddress(address) + for _, account := range accounts { + balanceStorageHelper.AddInterestingAddress(account.Address) } + log.Printf("construction tester initialized with %d accounts\n", len(accounts)) + // Load prefunded accounts var accountBalanceRequests []*utils.AccountBalanceRequest for _, prefundedAcc := range config.Construction.PrefundedAccounts { - address := prefundedAcc.Address accountBalance := &utils.AccountBalanceRequest{ - Account: &types.AccountIdentifier{ - Address: address, - }, + Account: prefundedAcc.AccountIdentifier, Network: network, Currency: prefundedAcc.Currency, } @@ -236,6 +234,7 @@ func InitializeConstruction( logger, cancel, []storage.BlockWorker{balanceStorage, coinStorage, broadcastStorage}, + syncer.DefaultCacheSize, config.MaxSyncConcurrency, ) diff --git a/pkg/tester/data.go b/pkg/tester/data.go index 97bd2ad1..25c7ba9e 100644 --- a/pkg/tester/data.go +++ b/pkg/tester/data.go @@ -30,6 +30,7 @@ import ( "github.com/coinbase/rosetta-sdk-go/reconciler" "github.com/coinbase/rosetta-sdk-go/statefulsyncer" "github.com/coinbase/rosetta-sdk-go/storage" + "github.com/coinbase/rosetta-sdk-go/syncer" "github.com/coinbase/rosetta-sdk-go/types" "github.com/coinbase/rosetta-sdk-go/utils" "github.com/fatih/color" @@ -263,6 +264,7 @@ func InitializeData( logger, cancel, blockWorkers, + syncer.DefaultCacheSize, config.MaxSyncConcurrency, ) @@ -641,7 +643,8 @@ func (t *DataTester) recursiveOpSearch( logger, cancel, []storage.BlockWorker{balanceStorage}, - t.config.SyncConcurrency, + syncer.DefaultCacheSize, + t.config.MaxSyncConcurrency, ) g, ctx := errgroup.WithContext(ctx) diff --git a/pkg/tester/data_results.go b/pkg/tester/data_results.go index 93696b0f..b1886dd7 100644 --- a/pkg/tester/data_results.go +++ b/pkg/tester/data_results.go @@ -25,6 +25,7 @@ import ( "github.com/coinbase/rosetta-cli/configuration" "github.com/coinbase/rosetta-cli/pkg/processor" + "github.com/coinbase/rosetta-sdk-go/asserter" "github.com/coinbase/rosetta-sdk-go/fetcher" "github.com/coinbase/rosetta-sdk-go/storage" "github.com/coinbase/rosetta-sdk-go/syncer" @@ -281,23 +282,15 @@ func (c *CheckDataTests) Print() { // indicating if all endpoints received // a non-500 response. func RequestResponseTest(err error) bool { - if errors.Is(err, fetcher.ErrExhaustedRetries) || errors.Is(err, fetcher.ErrRequestFailed) || - errors.Is(err, fetcher.ErrNoNetworks) || errors.Is(err, utils.ErrNetworkNotSupported) { - return false - } - - return true + return !(fetcher.Err(err) || errors.Is(err, utils.ErrNetworkNotSupported)) } // ResponseAssertionTest returns a boolean // indicating if all responses received from // the server were correctly formatted. func ResponseAssertionTest(err error) bool { - if errors.Is(err, fetcher.ErrAssertionFailed) { // nolint - return false - } - - return true + is, _ := asserter.Err(err) + return !is } // BlockSyncingTest returns a boolean diff --git a/pkg/tester/data_results_test.go b/pkg/tester/data_results_test.go index 25950f4e..15925308 100644 --- a/pkg/tester/data_results_test.go +++ b/pkg/tester/data_results_test.go @@ -24,6 +24,7 @@ import ( "github.com/coinbase/rosetta-cli/configuration" "github.com/coinbase/rosetta-cli/pkg/processor" + "github.com/coinbase/rosetta-sdk-go/asserter" "github.com/coinbase/rosetta-sdk-go/fetcher" "github.com/coinbase/rosetta-sdk-go/storage" "github.com/coinbase/rosetta-sdk-go/syncer" @@ -91,7 +92,7 @@ func TestComputeCheckDataResults(t *testing.T) { }, "default configuration, no storage, assertion errors": { cfg: configuration.DefaultConfiguration(), - err: []error{fetcher.ErrAssertionFailed}, + err: []error{asserter.ErrAmountValueMissing}, result: &CheckDataResults{ Tests: &CheckDataTests{ RequestResponse: true, From ee573d612c4bde29e6990581731305f0ef980737 Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Fri, 11 Sep 2020 10:29:43 -0700 Subject: [PATCH 007/376] Update version --- cmd/root.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/root.go b/cmd/root.go index a8fab93a..2939f840 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -132,6 +132,6 @@ var versionCmd = &cobra.Command{ Use: "version", Short: "Print rosetta-cli version", Run: func(cmd *cobra.Command, args []string) { - fmt.Println("v0.5.0") + fmt.Println("v0.5.1") }, } From bd018e27cff01a58fb743c2a8df7e1f17fa2bb2a Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Fri, 11 Sep 2020 10:32:21 -0700 Subject: [PATCH 008/376] Update README --- README.md | 8 ++++---- cmd/check_data.go | 8 ++++---- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index 2e2f216a..a04f5ac9 100644 --- a/README.md +++ b/README.md @@ -128,10 +128,10 @@ computed balance changes are equal to balance changes reported by the node. When re-running this command, it will start where it left off if you specify some data directory. Otherwise, it will create a new temporary directory and start again from the genesis block. If you want to discard some number of blocks -populate the --start flag with some block index. Starting from a given index -can be useful to debug a small range of blocks for issues but it is highly -recommended you sync from start to finish to ensure all correctness checks -are performed. +populate the start_index filed in the configuration file with some block index. +Starting from a given index can be useful to debug a small range of blocks for +issues but it is highly recommended you sync from start to finish to ensure +all correctness checks are performed. By default, account balances are looked up at specific heights (instead of only at the current block). If your node does not support this functionality diff --git a/cmd/check_data.go b/cmd/check_data.go index ac2e42d8..67567fd2 100644 --- a/cmd/check_data.go +++ b/cmd/check_data.go @@ -39,10 +39,10 @@ computed balance changes are equal to balance changes reported by the node. When re-running this command, it will start where it left off if you specify some data directory. Otherwise, it will create a new temporary directory and start again from the genesis block. If you want to discard some number of blocks -populate the --start flag with some block index. Starting from a given index -can be useful to debug a small range of blocks for issues but it is highly -recommended you sync from start to finish to ensure all correctness checks -are performed. +populate the start_index filed in the configuration file with some block index. +Starting from a given index can be useful to debug a small range of blocks for +issues but it is highly recommended you sync from start to finish to ensure +all correctness checks are performed. By default, account balances are looked up at specific heights (instead of only at the current block). If your node does not support this functionality From bfe5fc32804b5f75ed2478cd988528a85b12a5da Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Fri, 11 Sep 2020 10:45:48 -0700 Subject: [PATCH 009/376] Update config examples --- examples/configuration/bitcoin.json | 25 +++++++++++++------------ examples/configuration/default.json | 4 ++-- examples/configuration/ethereum.json | 23 ++++++++++++----------- examples/configuration/simple.json | 2 -- 4 files changed, 27 insertions(+), 27 deletions(-) diff --git a/examples/configuration/bitcoin.json b/examples/configuration/bitcoin.json index 86f6989f..ff109df9 100644 --- a/examples/configuration/bitcoin.json +++ b/examples/configuration/bitcoin.json @@ -7,13 +7,14 @@ "data_directory": "bitcoin-data", "http_timeout": 300, "retry_elapsed_time": 0, - "sync_concurrency": 0, - "transaction_concurrency": 0, + "max_online_connections": 0, + "max_sync_concurrency": 0, "tip_delay": 1800, "disable_memory_limit": true, "log_configuration": false, "construction": { "offline_url": "", + "max_offline_connections": 0, "stale_depth": 0, "broadcast_limit": 0, "ignore_broadcast_failures": false, @@ -27,7 +28,7 @@ "concurrency": 1, "scenarios": [ { - "name": "find_address", + "name": "find_account", "actions": [ { "input": "{\"symbol\":\"tBTC\", \"decimals\":8}", @@ -37,7 +38,7 @@ { "input": "{\"minimum_balance\":{\"value\": \"0\", \"currency\": {{currency}}}, \"create_limit\":1}", "type": "find_balance", - "output_path": "random_address" + "output_path": "random_account" } ] }, @@ -45,9 +46,9 @@ "name": "request", "actions": [ { - "input": "{\"address\": {{random_address.account.address}}, \"minimum_balance\":{\"value\": \"1000000\", \"currency\": {{currency}}}}", + "input": "{\"account_identifier\": {{random_account.account_identifier}}, \"minimum_balance\":{\"value\": \"1000000\", \"currency\": {{currency}}}}", "type": "find_balance", - "output_path": "loaded_address" + "output_path": "loaded_account" } ] } @@ -73,11 +74,11 @@ { "input": "{\"network_identifier\": {{network}}, \"public_key\": {{key.public_key}}}", "type": "derive", - "output_path": "address" + "output_path": "account" }, { - "input": "{\"address\": {{address.address}}, \"keypair\": {{key}}}", - "type": "save_address" + "input": "{\"account_identifier\": {{account.account_identifier}}, \"keypair\": {{key}}}", + "type": "save_account" } ] } @@ -159,7 +160,7 @@ "output_path": "sender_amount" }, { - "input": "{\"not_address\":[{{sender.account.address}}], \"not_coins\":[{{sender.coin}}], \"minimum_balance\":{\"value\": \"0\", \"currency\": {{currency}}}, \"create_limit\": 100, \"create_probability\": 50}", + "input": "{\"not_account_identifier\":[{{sender.account_identifier}}], \"not_coins\":[{{sender.coin}}], \"minimum_balance\":{\"value\": \"0\", \"currency\": {{currency}}}, \"create_limit\": 100, \"create_probability\": 50}", "type": "find_balance", "output_path": "recipient" }, @@ -169,7 +170,7 @@ "output_path": "transfer.confirmation_depth" }, { - "input": "[{\"operation_identifier\":{\"index\":0},\"type\":\"INPUT\",\"account\":{\"address\":{{sender.account.address}}},\"amount\":{\"value\":{{sender_amount}},\"currency\":{{currency}}}, \"coin_change\":{\"coin_action\":\"coin_spent\", \"coin_identifier\":{{sender.coin}}}},{\"operation_identifier\":{\"index\":1},\"type\":\"OUTPUT\",\"account\":{\"address\":{{recipient.account.address}}},\"amount\":{\"value\":{{recipient_amount}},\"currency\":{{currency}}}}, {\"operation_identifier\":{\"index\":2},\"type\":\"OUTPUT\",\"account\":{\"address\":{{sender.account.address}}},\"amount\":{\"value\":{{change_amount}},\"currency\":{{currency}}}}]", + "input": "[{\"operation_identifier\":{\"index\":0},\"type\":\"INPUT\",\"account\":{{sender.account_identifier}},\"amount\":{\"value\":{{sender_amount}},\"currency\":{{currency}}}, \"coin_change\":{\"coin_action\":\"coin_spent\", \"coin_identifier\":{{sender.coin}}}},{\"operation_identifier\":{\"index\":1},\"type\":\"OUTPUT\",\"account\":{{recipient.account_identifier}},\"amount\":{\"value\":{{recipient_amount}},\"currency\":{{currency}}}}, {\"operation_identifier\":{\"index\":2},\"type\":\"OUTPUT\",\"account\":{{sender.account_identifier}},\"amount\":{\"value\":{{change_amount}},\"currency\":{{currency}}}}]", "type": "set_variable", "output_path": "transfer.operations" }, @@ -209,4 +210,4 @@ }, "results_output_file": "" } -} +} \ No newline at end of file diff --git a/examples/configuration/default.json b/examples/configuration/default.json index 530cd1f8..54134586 100644 --- a/examples/configuration/default.json +++ b/examples/configuration/default.json @@ -7,8 +7,8 @@ "data_directory": "", "http_timeout": 10, "retry_elapsed_time": 60, - "sync_concurrency": 8, - "transaction_concurrency": 16, + "max_online_connections": 120, + "max_sync_concurrency": 64, "tip_delay": 300, "disable_memory_limit": false, "log_configuration": false, diff --git a/examples/configuration/ethereum.json b/examples/configuration/ethereum.json index 1b79d344..cf777e37 100644 --- a/examples/configuration/ethereum.json +++ b/examples/configuration/ethereum.json @@ -7,13 +7,14 @@ "data_directory": "ethereum-data", "http_timeout": 300, "retry_elapsed_time": 0, - "sync_concurrency": 0, - "transaction_concurrency": 0, + "max_online_connections": 0, + "max_sync_concurrency": 0, "tip_delay": 60, "disable_memory_limit": true, "log_configuration": false, "construction": { "offline_url": "", + "max_offline_connections": 0, "stale_depth": 0, "broadcast_limit": 0, "ignore_broadcast_failures": false, @@ -27,7 +28,7 @@ "concurrency": 1, "scenarios": [ { - "name": "find_address", + "name": "find_account", "actions": [ { "input": "{\"symbol\":\"ETH\", \"decimals\":18}", @@ -37,7 +38,7 @@ { "input": "{\"minimum_balance\":{\"value\": \"0\", \"currency\": {{currency}}}, \"create_limit\":1}", "type": "find_balance", - "output_path": "random_address" + "output_path": "random_account" } ] }, @@ -45,9 +46,9 @@ "name": "request", "actions": [ { - "input": "{\"address\": {{random_address.account.address}}, \"minimum_balance\":{\"value\": \"10000000000000000\", \"currency\": {{currency}}}}", + "input": "{\"account_identifier\": {{random_account.account_identifier}}, \"minimum_balance\":{\"value\": \"10000000000000000\", \"currency\": {{currency}}}}", "type": "find_balance", - "output_path": "loaded_address" + "output_path": "loaded_account" } ] } @@ -73,11 +74,11 @@ { "input": "{\"network_identifier\": {{network}}, \"public_key\": {{key.public_key}}}", "type": "derive", - "output_path": "address" + "output_path": "account" }, { - "input": "{\"address\": {{address.address}}, \"keypair\": {{key}}}", - "type": "save_address" + "input": "{\"account_identifier\": {{account.account_identifier}}, \"keypair\": {{key}}}", + "type": "save_account" } ] } @@ -130,7 +131,7 @@ "output_path": "sender_amount" }, { - "input": "{\"not_address\":[{{sender.account.address}}], \"minimum_balance\":{\"value\": \"0\", \"currency\": {{currency}}}, \"create_limit\": 100, \"create_probability\": 50}", + "input": "{\"not_account_identifier\":[{{sender.account_identifier}}], \"minimum_balance\":{\"value\": \"0\", \"currency\": {{currency}}}, \"create_limit\": 100, \"create_probability\": 50}", "type": "find_balance", "output_path": "recipient" }, @@ -140,7 +141,7 @@ "output_path": "transfer.confirmation_depth" }, { - "input": "[{\"operation_identifier\":{\"index\":0},\"type\":\"transfer\",\"account\":{\"address\":{{sender.account.address}}},\"amount\":{\"value\":{{sender_amount}},\"currency\":{{currency}}}},{\"operation_identifier\":{\"index\":1},\"type\":\"transfer\",\"account\":{\"address\":{{recipient.account.address}}},\"amount\":{\"value\":{{recipient_amount}},\"currency\":{{currency}}}}]", + "input": "[{\"operation_identifier\":{\"index\":0},\"type\":\"transfer\",\"account\":{{sender.account_identifier}},\"amount\":{\"value\":{{sender_amount}},\"currency\":{{currency}}}},{\"operation_identifier\":{\"index\":1},\"type\":\"transfer\",\"account\":{{recipient.account_identifier}},\"amount\":{\"value\":{{recipient_amount}},\"currency\":{{currency}}}}]", "type": "set_variable", "output_path": "transfer.operations" } diff --git a/examples/configuration/simple.json b/examples/configuration/simple.json index 90b21c8f..1237cc71 100644 --- a/examples/configuration/simple.json +++ b/examples/configuration/simple.json @@ -6,8 +6,6 @@ "online_url": "http://localhost:8080", "data_directory": "", "http_timeout": 10, - "sync_concurrency": 8, - "transaction_concurrency": 16, "tip_delay": 300, "data": { "historical_balance_disabled": true, From 6a7bef220df7d500e1a593c0102866e19e30a729 Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Fri, 11 Sep 2020 10:57:49 -0700 Subject: [PATCH 010/376] use tiny cache for tests --- pkg/tester/data_results_test.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/pkg/tester/data_results_test.go b/pkg/tester/data_results_test.go index 15925308..c40ab100 100644 --- a/pkg/tester/data_results_test.go +++ b/pkg/tester/data_results_test.go @@ -306,7 +306,11 @@ func TestComputeCheckDataResults(t *testing.T) { assert.NoError(t, err) ctx := context.Background() - localStore, err := storage.NewBadgerStorage(ctx, dir) + localStore, err := storage.NewBadgerStorage( + ctx, + dir, + storage.WithIndexCacheSize(storage.TinyIndexCacheSize), + ) assert.NoError(t, err) logPath := path.Join(dir, "results.json") From 38dccdb1854338ab08a17aa59bbdc452e992ae00 Mon Sep 17 00:00:00 2001 From: Cindy Kuang Date: Fri, 11 Sep 2020 17:32:53 -0700 Subject: [PATCH 011/376] updated error handling and fixed tests --- go.sum | 4 +++ pkg/processor/coordinator_helper.go | 7 +++++- pkg/tester/data_results.go | 39 ++++++++++++++++------------- pkg/tester/data_results_test.go | 25 ++++++++++++++++++ 4 files changed, 56 insertions(+), 19 deletions(-) diff --git a/go.sum b/go.sum index c468c1f6..3f7aace2 100644 --- a/go.sum +++ b/go.sum @@ -352,6 +352,7 @@ github.com/rs/xhandler v0.0.0-20160618193221-ed27b6fd6521/go.mod h1:RvLn4FgxWubr github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/segmentio/golines v0.0.0-20200808004416-0a9796b248e8/go.mod h1:bQSh5qdVR67XiCKbaVvYO41s50c5hQo+3cY/1CQQ3xQ= +github.com/segmentio/golines v0.0.0-20200824192126-7f30d3046793 h1:rhR7esJSmty+9ST6Gsp7mlQHkpISw2DiYjuFaz3dRDg= github.com/segmentio/golines v0.0.0-20200824192126-7f30d3046793/go.mod h1:bQSh5qdVR67XiCKbaVvYO41s50c5hQo+3cY/1CQQ3xQ= github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= @@ -446,6 +447,7 @@ golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 h1:psW17arqaxU48Z5kZ0CQnk golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200728195943-123391ffb6de h1:ikNHVSjEfnvz6sxdSPCaPt572qowuyMDMJLLm3Db3ig= golang.org/x/crypto v0.0.0-20200728195943-123391ffb6de/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20200820211705-5c72a883971a h1:vclmkQCjlDX5OydZ9wv8rBCcS0QyQY66Mpf/7BZbInM= golang.org/x/crypto v0.0.0-20200820211705-5c72a883971a/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= @@ -454,6 +456,7 @@ golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3 h1:XQyxROzUlZH+WIQwySDgnISg golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.3.0 h1:RM4zey1++hCTbCVQfnWeKs9/IEsaBLA8vTkd0WVtmH4= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/net v0.0.0-20180719180050-a680a1efc54d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -535,6 +538,7 @@ golang.org/x/tools v0.0.0-20191130070609-6e064ea0cf2d/go.mod h1:b+2E5dAYhXwXZwtn golang.org/x/tools v0.0.0-20200626171337-aa94e735be7f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200729181040-64cdafbe085c/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20200731060945-b5fad4ed8dd6/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20200904185747-39188db58858 h1:xLt+iB5ksWcZVxqc+g9K41ZHy+6MKWfXCDsjSThnsPA= golang.org/x/tools v0.0.0-20200904185747-39188db58858/go.mod h1:Cj7w3i3Rnn0Xh82ur9kSqwfTHTeVxaDqrfMjpcNT6bE= golang.org/x/tools/gopls v0.4.4/go.mod h1:zhyGzA+CAtREUwwq/btQxEx2FHnGzDwJvGs5YqdVCbE= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= diff --git a/pkg/processor/coordinator_helper.go b/pkg/processor/coordinator_helper.go index 5935d90d..8d31138f 100644 --- a/pkg/processor/coordinator_helper.go +++ b/pkg/processor/coordinator_helper.go @@ -259,7 +259,12 @@ func (c *CoordinatorHelper) StoreKey( // We optimisically add the interesting address although the dbTx could be reverted. c.balanceStorageHelper.AddInterestingAddress(account.Address) - _, _ = c.counterStorage.UpdateTransactional(ctx, dbTx, storage.AddressesCreatedCounter, big.NewInt(1)) + _, _ = c.counterStorage.UpdateTransactional( + ctx, + dbTx, + storage.AddressesCreatedCounter, + big.NewInt(1), + ) return c.keyStorage.StoreTransactional(ctx, account, keyPair, dbTx) } diff --git a/pkg/tester/data_results.go b/pkg/tester/data_results.go index b1886dd7..e97374fe 100644 --- a/pkg/tester/data_results.go +++ b/pkg/tester/data_results.go @@ -65,8 +65,10 @@ func (c *CheckDataResults) Print() { } fmt.Printf("\n") - c.Tests.Print() - fmt.Printf("\n") + if c.Tests != nil { + c.Tests.Print() + fmt.Printf("\n") + } if c.Stats != nil { c.Stats.Print() fmt.Printf("\n") @@ -297,18 +299,11 @@ func ResponseAssertionTest(err error) bool { // indicating if it was possible to sync // blocks. func BlockSyncingTest(err error, blocksSynced bool) *bool { - relatedErrors := []error{ - syncer.ErrCannotRemoveGenesisBlock, - syncer.ErrOutOfOrder, - storage.ErrDuplicateKey, - storage.ErrDuplicateTransactionHash, - } syncPass := true - for _, relatedError := range relatedErrors { - if errors.Is(err, relatedError) { - syncPass = false - break - } + storageFailed, _ := storage.Err(err) + if syncer.Err(err) || + (storageFailed && !errors.Is(err, storage.ErrNegativeBalance)) { + syncPass = false } if !blocksSynced && syncPass { @@ -322,12 +317,9 @@ func BlockSyncingTest(err error, blocksSynced bool) *bool { // indicating if any balances went negative // while syncing. func BalanceTrackingTest(cfg *configuration.Configuration, err error, operationsSeen bool) *bool { - relatedErrors := []error{ - storage.ErrNegativeBalance, - } balancePass := true - for _, relatedError := range relatedErrors { - if errors.Is(err, relatedError) { + for _, balangeStorageErr := range storage.BalanceStorageErrs { + if errors.Is(err, balangeStorageErr) { balancePass = false break } @@ -431,6 +423,17 @@ func ComputeCheckDataResults( if err != nil { results.Error = err.Error() + // If all tests pass, but we still encountered an error, + // then we hard exit without showing check:data results + // because the error falls beyond our test coverage. + if tests.RequestResponse && + tests.ResponseAssertion && + (tests.BlockSyncing == nil || *tests.BlockSyncing) && + (tests.BalanceTracking == nil || *tests.BalanceTracking) && + (tests.Reconciliation == nil || *tests.Reconciliation) { + results.Tests = nil + } + // We never want to populate an end condition // if there was an error! return results diff --git a/pkg/tester/data_results_test.go b/pkg/tester/data_results_test.go index c40ab100..465266e4 100644 --- a/pkg/tester/data_results_test.go +++ b/pkg/tester/data_results_test.go @@ -16,6 +16,7 @@ package tester import ( "context" + "errors" "fmt" "math/big" "path" @@ -289,6 +290,30 @@ func TestComputeCheckDataResults(t *testing.T) { }, }, }, + "default configuration, no storage, unknown errors": { + cfg: configuration.DefaultConfiguration(), + err: []error{errors.New("unsure how to handle this error")}, + result: &CheckDataResults{}, + }, + "default configuration, counter storage no blocks, unknown errors": { + cfg: configuration.DefaultConfiguration(), + provideCounterStorage: true, + err: []error{errors.New("unsure how to handle this error")}, + result: &CheckDataResults{ + Stats: &CheckDataStats{}, + }, + }, + "default configuration, counter storage with blocks, unknown errors": { + cfg: configuration.DefaultConfiguration(), + provideCounterStorage: true, + blockCount: 100, + err: []error{errors.New("unsure how to handle this error")}, + result: &CheckDataResults{ + Stats: &CheckDataStats{ + Blocks: 100, + }, + }, + }, } for name, test := range tests { From dd8198896ed5ce39feca39a172da3f67d6aa5c81 Mon Sep 17 00:00:00 2001 From: Cindy Kuang Date: Fri, 11 Sep 2020 17:43:05 -0700 Subject: [PATCH 012/376] nit --- pkg/tester/data_results.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/tester/data_results.go b/pkg/tester/data_results.go index e97374fe..9fe79c74 100644 --- a/pkg/tester/data_results.go +++ b/pkg/tester/data_results.go @@ -318,8 +318,8 @@ func BlockSyncingTest(err error, blocksSynced bool) *bool { // while syncing. func BalanceTrackingTest(cfg *configuration.Configuration, err error, operationsSeen bool) *bool { balancePass := true - for _, balangeStorageErr := range storage.BalanceStorageErrs { - if errors.Is(err, balangeStorageErr) { + for _, balanceStorageErr := range storage.BalanceStorageErrs { + if errors.Is(err, balanceStorageErr) { balancePass = false break } From 066047665e3cb4db0088f21b224d016dca382a17 Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Fri, 11 Sep 2020 18:14:26 -0700 Subject: [PATCH 013/376] Update rosetta-sdk-go --- .circleci/config.yml | 2 +- README.md | 6 ++++++ go.mod | 2 +- go.sum | 2 ++ 4 files changed, 10 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 79097af3..d0c43111 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -50,7 +50,7 @@ jobs: name: default steps: - *fast-checkout - - run: curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(go env GOPATH)/bin v1.24.0 + - run: curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(go env GOPATH)/bin v1.30.0 - run: make lint check-license: executor: diff --git a/README.md b/README.md index a04f5ac9..0300c5d0 100644 --- a/README.md +++ b/README.md @@ -38,6 +38,12 @@ The binary will be installed inside the `./bin` directory (relative to where the _Downloading binaries from the Github UI will cause permission errors on Mac._ +### Installing in Custom Location +To download the binary into a specific directory, run: +``` +curl -sSfL https://raw.githubusercontent.com/coinbase/rosetta-cli/master/scripts/install.sh | sh -s -- -b +``` + ## Usage ``` CLI for the Rosetta API diff --git a/go.mod b/go.mod index 14db5cb9..e7b82bb3 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/coinbase/rosetta-cli go 1.13 require ( - github.com/coinbase/rosetta-sdk-go v0.4.2-0.20200911162523-bbfbd83145d4 + github.com/coinbase/rosetta-sdk-go v0.4.2-0.20200912010944-f7d13f597014 github.com/fatih/color v1.9.0 github.com/jinzhu/copier v0.0.0-20190924061706-b57f9002281a github.com/olekukonko/tablewriter v0.0.2-0.20190409134802-7e037d187b0c diff --git a/go.sum b/go.sum index 3f7aace2..f3d7feec 100644 --- a/go.sum +++ b/go.sum @@ -125,6 +125,8 @@ github.com/coinbase/rosetta-sdk-go v0.4.1 h1:bL8qgwRZ6O56V+drjbdRAhnPZYdkNFjmY5V github.com/coinbase/rosetta-sdk-go v0.4.1/go.mod h1:Luv0AhzZH81eul2hYZ3w0hBGwmFPiexwbntYxihEZck= github.com/coinbase/rosetta-sdk-go v0.4.2-0.20200911162523-bbfbd83145d4 h1:aEQB1Y2qp036Cfy+Cm2pjtPEKmFofw/QKzfd3JL/wIU= github.com/coinbase/rosetta-sdk-go v0.4.2-0.20200911162523-bbfbd83145d4/go.mod h1:Luv0AhzZH81eul2hYZ3w0hBGwmFPiexwbntYxihEZck= +github.com/coinbase/rosetta-sdk-go v0.4.2-0.20200912010944-f7d13f597014 h1:vMWAMtcGnpEhWh6kEDzDqviivHwFRp9w01MXGCLJCMU= +github.com/coinbase/rosetta-sdk-go v0.4.2-0.20200912010944-f7d13f597014/go.mod h1:Luv0AhzZH81eul2hYZ3w0hBGwmFPiexwbntYxihEZck= github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk= From 936f5937f4d48314441169f54c759f42e9a673f6 Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Fri, 11 Sep 2020 18:30:27 -0700 Subject: [PATCH 014/376] Infer historical balance lookup --- configuration/configuration.go | 4 +- configuration/configuration_test.go | 15 +++--- pkg/tester/data.go | 83 +++++++++++++++++------------ 3 files changed, 59 insertions(+), 43 deletions(-) diff --git a/configuration/configuration.go b/configuration/configuration.go index 9a63676e..fa8356bb 100644 --- a/configuration/configuration.go +++ b/configuration/configuration.go @@ -230,11 +230,11 @@ type DataConfiguration struct { // it will be ignored. BootstrapBalances string `json:"bootstrap_balances"` - // HistoricalBalanceDisabled is a boolean that dictates how balance lookup is performed. + // HistoricalBalanceEnabled is a boolean that dictates how balance lookup is performed. // When set to true, balances are looked up at the block where a balance // change occurred instead of at the current block. Blockchains that do not support // historical balance lookup should set this to false. - HistoricalBalanceDisabled bool `json:"historical_balance_disabled"` + HistoricalBalanceEnabled *bool `json:"historical_balance_enabled"` // InterestingAccounts is a path to a file listing all accounts to check on each block. Look // at the examples directory for an example of how to structure this file. diff --git a/configuration/configuration_test.go b/configuration/configuration_test.go index 0fae2208..2994a869 100644 --- a/configuration/configuration_test.go +++ b/configuration/configuration_test.go @@ -27,12 +27,13 @@ import ( ) var ( - startIndex = int64(89) - badStartIndex = int64(-10) - goodCoverage = float64(0.33) - badCoverage = float64(-2) - endTip = false - fakeWorkflows = []*job.Workflow{ + startIndex = int64(89) + badStartIndex = int64(-10) + goodCoverage = float64(0.33) + badCoverage = float64(-2) + endTip = false + historicalEnabled = true + fakeWorkflows = []*job.Workflow{ { Name: string(job.CreateAccount), Concurrency: job.ReservedWorkflowConcurrency, @@ -72,7 +73,7 @@ var ( InactiveReconciliationConcurrency: 2938, InactiveReconciliationFrequency: 3, ReconciliationDisabled: false, - HistoricalBalanceDisabled: true, + HistoricalBalanceEnabled: &historicalEnabled, StartIndex: &startIndex, EndConditions: &DataEndConditions{ ReconciliationCoverage: &goodCoverage, diff --git a/pkg/tester/data.go b/pkg/tester/data.go index 25c7ba9e..ab618a4f 100644 --- a/pkg/tester/data.go +++ b/pkg/tester/data.go @@ -64,20 +64,21 @@ const ( // DataTester coordinates the `check:data` test. type DataTester struct { - network *types.NetworkIdentifier - database storage.Database - config *configuration.Configuration - syncer *statefulsyncer.StatefulSyncer - reconciler *reconciler.Reconciler - logger *logger.Logger - balanceStorage *storage.BalanceStorage - blockStorage *storage.BlockStorage - counterStorage *storage.CounterStorage - reconcilerHandler *processor.ReconcilerHandler - fetcher *fetcher.Fetcher - signalReceived *bool - genesisBlock *types.BlockIdentifier - cancel context.CancelFunc + network *types.NetworkIdentifier + database storage.Database + config *configuration.Configuration + syncer *statefulsyncer.StatefulSyncer + reconciler *reconciler.Reconciler + logger *logger.Logger + balanceStorage *storage.BalanceStorage + blockStorage *storage.BlockStorage + counterStorage *storage.CounterStorage + reconcilerHandler *processor.ReconcilerHandler + fetcher *fetcher.Fetcher + signalReceived *bool + genesisBlock *types.BlockIdentifier + cancel context.CancelFunc + historicalBalanceEnabled bool endCondition configuration.CheckDataEndCondition endConditionDetail string @@ -197,12 +198,25 @@ func InitializeData( log.Fatalf("%s: unable to get previously seen accounts", err.Error()) } + // Determine if we should perform historical balance lookups + var historicalBalanceEnabled bool + if config.Data.HistoricalBalanceEnabled != nil { + historicalBalanceEnabled = *config.Data.HistoricalBalanceEnabled + } else { // we must look it up + networkOptions, fetchErr := fetcher.NetworkOptionsRetry(ctx, network, nil) + if err != nil { + log.Fatalf("%s: unable to get network options", fetchErr.Err.Error()) + } + + historicalBalanceEnabled = networkOptions.Allow.HistoricalBalanceLookup + } + r := reconciler.New( reconcilerHelper, reconcilerHandler, reconciler.WithActiveConcurrency(int(config.Data.ActiveReconciliationConcurrency)), reconciler.WithInactiveConcurrency(int(config.Data.InactiveReconciliationConcurrency)), - reconciler.WithLookupBalanceByBlock(!config.Data.HistoricalBalanceDisabled), + reconciler.WithLookupBalanceByBlock(historicalBalanceEnabled), reconciler.WithInterestingAccounts(interestingAccounts), reconciler.WithSeenAccounts(seenAccounts), reconciler.WithDebugLogging(config.Data.LogReconciliations), @@ -214,7 +228,7 @@ func InitializeData( balanceStorageHelper := processor.NewBalanceStorageHelper( network, fetcher, - !config.Data.HistoricalBalanceDisabled, + historicalBalanceEnabled, exemptAccounts, false, ) @@ -269,20 +283,21 @@ func InitializeData( ) return &DataTester{ - network: network, - database: localStore, - config: config, - syncer: syncer, - cancel: cancel, - reconciler: r, - logger: logger, - balanceStorage: balanceStorage, - blockStorage: blockStorage, - counterStorage: counterStorage, - reconcilerHandler: reconcilerHandler, - fetcher: fetcher, - signalReceived: signalReceived, - genesisBlock: genesisBlock, + network: network, + database: localStore, + config: config, + syncer: syncer, + cancel: cancel, + reconciler: r, + logger: logger, + balanceStorage: balanceStorage, + blockStorage: blockStorage, + counterStorage: counterStorage, + reconcilerHandler: reconcilerHandler, + fetcher: fetcher, + signalReceived: signalReceived, + genesisBlock: genesisBlock, + historicalBalanceEnabled: historicalBalanceEnabled, } } @@ -501,9 +516,9 @@ func (t *DataTester) HandleErr(ctx context.Context, err error, sigListeners []co ExitData(t.config, t.counterStorage, t.balanceStorage, err, 1, "", "") } - if t.config.Data.HistoricalBalanceDisabled { + if !t.historicalBalanceEnabled { color.Yellow( - "Can't find the block missing operations automatically, please enable --lookup-balance-by-block", + "Can't find the block missing operations automatically, please enable historical balance lookup", ) ExitData(t.config, t.counterStorage, t.balanceStorage, err, 1, "", "") } @@ -613,14 +628,14 @@ func (t *DataTester) recursiveOpSearch( // Do not do any inactive lookups when looking for the block with missing // operations. reconciler.WithInactiveConcurrency(0), - reconciler.WithLookupBalanceByBlock(!t.config.Data.HistoricalBalanceDisabled), + reconciler.WithLookupBalanceByBlock(t.historicalBalanceEnabled), reconciler.WithInterestingAccounts([]*reconciler.AccountCurrency{accountCurrency}), ) balanceStorageHelper := processor.NewBalanceStorageHelper( t.network, t.fetcher, - !t.config.Data.HistoricalBalanceDisabled, + t.historicalBalanceEnabled, nil, false, ) From 62e41ab2405c82bcffccbf73c30d5430571d52cd Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Fri, 11 Sep 2020 18:34:32 -0700 Subject: [PATCH 015/376] Update examples --- configuration/configuration.go | 2 +- examples/configuration/bitcoin.json | 3 +-- examples/configuration/default.json | 1 - examples/configuration/ethereum.json | 3 +-- examples/configuration/simple.json | 2 +- 5 files changed, 4 insertions(+), 7 deletions(-) diff --git a/configuration/configuration.go b/configuration/configuration.go index fa8356bb..fde818db 100644 --- a/configuration/configuration.go +++ b/configuration/configuration.go @@ -234,7 +234,7 @@ type DataConfiguration struct { // When set to true, balances are looked up at the block where a balance // change occurred instead of at the current block. Blockchains that do not support // historical balance lookup should set this to false. - HistoricalBalanceEnabled *bool `json:"historical_balance_enabled"` + HistoricalBalanceEnabled *bool `json:"historical_balance_enabled,omitempty"` // InterestingAccounts is a path to a file listing all accounts to check on each block. Look // at the examples directory for an example of how to structure this file. diff --git a/examples/configuration/bitcoin.json b/examples/configuration/bitcoin.json index ff109df9..0b6750ef 100644 --- a/examples/configuration/bitcoin.json +++ b/examples/configuration/bitcoin.json @@ -199,7 +199,6 @@ "ignore_reconciliation_error": false, "exempt_accounts": "", "bootstrap_balances": "", - "historical_balance_disabled": true, "interesting_accounts": "", "reconciliation_disabled": false, "inactive_discrepency_search_disabled": false, @@ -210,4 +209,4 @@ }, "results_output_file": "" } -} \ No newline at end of file +} diff --git a/examples/configuration/default.json b/examples/configuration/default.json index 54134586..badbe812 100644 --- a/examples/configuration/default.json +++ b/examples/configuration/default.json @@ -24,7 +24,6 @@ "ignore_reconciliation_error": false, "exempt_accounts": "", "bootstrap_balances": "", - "historical_balance_disabled": false, "interesting_accounts": "", "reconciliation_disabled": false, "inactive_discrepency_search_disabled": false, diff --git a/examples/configuration/ethereum.json b/examples/configuration/ethereum.json index cf777e37..0a6ef244 100644 --- a/examples/configuration/ethereum.json +++ b/examples/configuration/ethereum.json @@ -166,7 +166,6 @@ "ignore_reconciliation_error": false, "exempt_accounts": "", "bootstrap_balances": "", - "historical_balance_disabled": false, "interesting_accounts": "", "reconciliation_disabled": false, "inactive_discrepency_search_disabled": false, @@ -177,4 +176,4 @@ }, "results_output_file": "" } -} \ No newline at end of file +} diff --git a/examples/configuration/simple.json b/examples/configuration/simple.json index 1237cc71..cbf87552 100644 --- a/examples/configuration/simple.json +++ b/examples/configuration/simple.json @@ -8,7 +8,7 @@ "http_timeout": 10, "tip_delay": 300, "data": { - "historical_balance_disabled": true, + "historical_balance_enabled": false, "reconciliation_disabled": true, "inactive_discrepency_search_disabled": true, "balance_tracking_disabled": true, From c564e63f8a5175292e4940b68a7842a0f7a07d45 Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Mon, 14 Sep 2020 07:39:15 -0700 Subject: [PATCH 016/376] Upgrade to new rosetta-sdk-go release --- go.mod | 2 +- go.sum | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index e7b82bb3..8e93ca1c 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/coinbase/rosetta-cli go 1.13 require ( - github.com/coinbase/rosetta-sdk-go v0.4.2-0.20200912010944-f7d13f597014 + github.com/coinbase/rosetta-sdk-go v0.4.2 github.com/fatih/color v1.9.0 github.com/jinzhu/copier v0.0.0-20190924061706-b57f9002281a github.com/olekukonko/tablewriter v0.0.2-0.20190409134802-7e037d187b0c diff --git a/go.sum b/go.sum index f3d7feec..50119499 100644 --- a/go.sum +++ b/go.sum @@ -127,6 +127,8 @@ github.com/coinbase/rosetta-sdk-go v0.4.2-0.20200911162523-bbfbd83145d4 h1:aEQB1 github.com/coinbase/rosetta-sdk-go v0.4.2-0.20200911162523-bbfbd83145d4/go.mod h1:Luv0AhzZH81eul2hYZ3w0hBGwmFPiexwbntYxihEZck= github.com/coinbase/rosetta-sdk-go v0.4.2-0.20200912010944-f7d13f597014 h1:vMWAMtcGnpEhWh6kEDzDqviivHwFRp9w01MXGCLJCMU= github.com/coinbase/rosetta-sdk-go v0.4.2-0.20200912010944-f7d13f597014/go.mod h1:Luv0AhzZH81eul2hYZ3w0hBGwmFPiexwbntYxihEZck= +github.com/coinbase/rosetta-sdk-go v0.4.2 h1:ojiNdDmmMwXIVIRN9vNCPytXtRBAm+EGCibJMhQ5z8o= +github.com/coinbase/rosetta-sdk-go v0.4.2/go.mod h1:Luv0AhzZH81eul2hYZ3w0hBGwmFPiexwbntYxihEZck= github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk= From 527710701bbf41324e6bbdedb1f7acb2f282a966 Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Mon, 14 Sep 2020 07:46:28 -0700 Subject: [PATCH 017/376] Address breaking changes --- configuration/configuration.go | 4 ---- pkg/tester/construction.go | 6 +----- pkg/tester/data.go | 12 ++---------- 3 files changed, 3 insertions(+), 19 deletions(-) diff --git a/configuration/configuration.go b/configuration/configuration.go index fde818db..778f9272 100644 --- a/configuration/configuration.go +++ b/configuration/configuration.go @@ -309,10 +309,6 @@ type Configuration struct { // we are considered to be behind tip. TipDelay int64 `json:"tip_delay"` - // DisableMemoryLimit uses a performance-optimized database mode - // that uses more memory. - DisableMemoryLimit bool `json:"disable_memory_limit"` - // LogConfiguration determines if the configuration settings // should be printed to the console when a file is loaded. LogConfiguration bool `json:"log_configuration"` diff --git a/pkg/tester/construction.go b/pkg/tester/construction.go index e6ef0f44..0bb251dd 100644 --- a/pkg/tester/construction.go +++ b/pkg/tester/construction.go @@ -78,11 +78,7 @@ func InitializeConstruction( log.Fatalf("%s: cannot create command path", err.Error()) } - opts := []storage.BadgerOption{} - if !config.DisableMemoryLimit { - opts = append(opts, storage.WithMemoryLimit()) - } - localStore, err := storage.NewBadgerStorage(ctx, dataPath, opts...) + localStore, err := storage.NewBadgerStorage(ctx, dataPath) if err != nil { log.Fatalf("%s: unable to initialize database", err.Error()) } diff --git a/pkg/tester/data.go b/pkg/tester/data.go index ab618a4f..b1b12f6b 100644 --- a/pkg/tester/data.go +++ b/pkg/tester/data.go @@ -141,11 +141,7 @@ func InitializeData( log.Fatalf("%s: cannot create command path", err.Error()) } - opts := []storage.BadgerOption{} - if !config.DisableMemoryLimit { - opts = append(opts, storage.WithMemoryLimit()) - } - localStore, err := storage.NewBadgerStorage(ctx, dataPath, opts...) + localStore, err := storage.NewBadgerStorage(ctx, dataPath) if err != nil { log.Fatalf("%s: unable to initialize database", err.Error()) } @@ -580,11 +576,7 @@ func (t *DataTester) recursiveOpSearch( } defer utils.RemoveTempDir(tmpDir) - opts := []storage.BadgerOption{} - if !t.config.DisableMemoryLimit { - opts = append(opts, storage.WithMemoryLimit()) - } - localStore, err := storage.NewBadgerStorage(ctx, tmpDir, opts...) + localStore, err := storage.NewBadgerStorage(ctx, tmpDir) if err != nil { return nil, fmt.Errorf("%w: unable to initialize database", err) } From 209dc5421e6ce5a46b57384283568d568d09521d Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Mon, 14 Sep 2020 07:53:51 -0700 Subject: [PATCH 018/376] Update config files --- examples/configuration/bitcoin.json | 3 +-- examples/configuration/default.json | 1 - examples/configuration/ethereum.json | 3 +-- 3 files changed, 2 insertions(+), 5 deletions(-) diff --git a/examples/configuration/bitcoin.json b/examples/configuration/bitcoin.json index 0b6750ef..a077cec3 100644 --- a/examples/configuration/bitcoin.json +++ b/examples/configuration/bitcoin.json @@ -10,7 +10,6 @@ "max_online_connections": 0, "max_sync_concurrency": 0, "tip_delay": 1800, - "disable_memory_limit": true, "log_configuration": false, "construction": { "offline_url": "", @@ -209,4 +208,4 @@ }, "results_output_file": "" } -} +} \ No newline at end of file diff --git a/examples/configuration/default.json b/examples/configuration/default.json index badbe812..7c5c722d 100644 --- a/examples/configuration/default.json +++ b/examples/configuration/default.json @@ -10,7 +10,6 @@ "max_online_connections": 120, "max_sync_concurrency": 64, "tip_delay": 300, - "disable_memory_limit": false, "log_configuration": false, "construction": null, "data": { diff --git a/examples/configuration/ethereum.json b/examples/configuration/ethereum.json index 0a6ef244..7d8d01e0 100644 --- a/examples/configuration/ethereum.json +++ b/examples/configuration/ethereum.json @@ -10,7 +10,6 @@ "max_online_connections": 0, "max_sync_concurrency": 0, "tip_delay": 60, - "disable_memory_limit": true, "log_configuration": false, "construction": { "offline_url": "", @@ -176,4 +175,4 @@ }, "results_output_file": "" } -} +} \ No newline at end of file From dd5277e3343e9d638a7e4f2b39f6d1578d6c24f0 Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Mon, 14 Sep 2020 08:17:19 -0700 Subject: [PATCH 019/376] Add more errors to request response --- pkg/tester/data_results.go | 5 ++++- pkg/tester/data_results_test.go | 14 ++++++++++++++ 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/pkg/tester/data_results.go b/pkg/tester/data_results.go index 9fe79c74..06252dad 100644 --- a/pkg/tester/data_results.go +++ b/pkg/tester/data_results.go @@ -284,7 +284,10 @@ func (c *CheckDataTests) Print() { // indicating if all endpoints received // a non-500 response. func RequestResponseTest(err error) bool { - return !(fetcher.Err(err) || errors.Is(err, utils.ErrNetworkNotSupported)) + return !(fetcher.Err(err) || + errors.Is(err, utils.ErrNetworkNotSupported) || + errors.Is(err, syncer.ErrGetNetworkStatusFailed) || + errors.Is(err, syncer.ErrFetchBlockFailed)) } // ResponseAssertionTest returns a boolean diff --git a/pkg/tester/data_results_test.go b/pkg/tester/data_results_test.go index 465266e4..11d8f653 100644 --- a/pkg/tester/data_results_test.go +++ b/pkg/tester/data_results_test.go @@ -91,6 +91,20 @@ func TestComputeCheckDataResults(t *testing.T) { }, }, }, + "default configuration, no storage, syncer and fetch errors": { + cfg: configuration.DefaultConfiguration(), + err: []error{ + syncer.ErrGetNetworkStatusFailed, + syncer.ErrFetchBlockFailed, + }, + result: &CheckDataResults{ + Tests: &CheckDataTests{ + RequestResponse: false, + ResponseAssertion: true, + BlockSyncing: &f, + }, + }, + }, "default configuration, no storage, assertion errors": { cfg: configuration.DefaultConfiguration(), err: []error{asserter.ErrAmountValueMissing}, From 2766be4eef7f1d7958d2ffc6fcbeff9cf414e285 Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Mon, 14 Sep 2020 10:34:31 -0700 Subject: [PATCH 020/376] [Fetcher] handle transient errors --- go.mod | 2 +- go.sum | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index 8e93ca1c..3a5682df 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/coinbase/rosetta-cli go 1.13 require ( - github.com/coinbase/rosetta-sdk-go v0.4.2 + github.com/coinbase/rosetta-sdk-go v0.4.3-0.20200914173330-69c73d3ff87a github.com/fatih/color v1.9.0 github.com/jinzhu/copier v0.0.0-20190924061706-b57f9002281a github.com/olekukonko/tablewriter v0.0.2-0.20190409134802-7e037d187b0c diff --git a/go.sum b/go.sum index 50119499..b60c4668 100644 --- a/go.sum +++ b/go.sum @@ -129,6 +129,8 @@ github.com/coinbase/rosetta-sdk-go v0.4.2-0.20200912010944-f7d13f597014 h1:vMWAM github.com/coinbase/rosetta-sdk-go v0.4.2-0.20200912010944-f7d13f597014/go.mod h1:Luv0AhzZH81eul2hYZ3w0hBGwmFPiexwbntYxihEZck= github.com/coinbase/rosetta-sdk-go v0.4.2 h1:ojiNdDmmMwXIVIRN9vNCPytXtRBAm+EGCibJMhQ5z8o= github.com/coinbase/rosetta-sdk-go v0.4.2/go.mod h1:Luv0AhzZH81eul2hYZ3w0hBGwmFPiexwbntYxihEZck= +github.com/coinbase/rosetta-sdk-go v0.4.3-0.20200914173330-69c73d3ff87a h1:lEI5Uy/qwhjzg+DCWJG94I+k3J+iG/zs6xjowmdzBvE= +github.com/coinbase/rosetta-sdk-go v0.4.3-0.20200914173330-69c73d3ff87a/go.mod h1:Luv0AhzZH81eul2hYZ3w0hBGwmFPiexwbntYxihEZck= github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk= From 657483b103c556432a48117d52d48d4f527ef6ff Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Mon, 14 Sep 2020 11:09:50 -0700 Subject: [PATCH 021/376] Update README with new config option --- README.md | 6 +++--- cmd/check_data.go | 6 +++--- go.mod | 2 +- go.sum | 2 ++ 4 files changed, 9 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index 0300c5d0..b4b83f06 100644 --- a/README.md +++ b/README.md @@ -140,9 +140,9 @@ issues but it is highly recommended you sync from start to finish to ensure all correctness checks are performed. By default, account balances are looked up at specific heights (instead of -only at the current block). If your node does not support this functionality -set historical balance disabled to true. This will make reconciliation much -less efficient but it will still work. +only at the current block). If your node does not support this functionality, +you can disable historical balance lookups in your configuration file. This will +make reconciliation much less efficient but it will still work. If check fails due to an INACTIVE reconciliation error (balance changed without any corresponding operation), the cli will automatically try to find the block diff --git a/cmd/check_data.go b/cmd/check_data.go index 67567fd2..53c92585 100644 --- a/cmd/check_data.go +++ b/cmd/check_data.go @@ -45,9 +45,9 @@ issues but it is highly recommended you sync from start to finish to ensure all correctness checks are performed. By default, account balances are looked up at specific heights (instead of -only at the current block). If your node does not support this functionality -set historical balance disabled to true. This will make reconciliation much -less efficient but it will still work. +only at the current block). If your node does not support this functionality, +you can disable historical balance lookups in your configuration file. This will +make reconciliation much less efficient but it will still work. If check fails due to an INACTIVE reconciliation error (balance changed without any corresponding operation), the cli will automatically try to find the block diff --git a/go.mod b/go.mod index 3a5682df..ce5d1b7d 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/coinbase/rosetta-cli go 1.13 require ( - github.com/coinbase/rosetta-sdk-go v0.4.3-0.20200914173330-69c73d3ff87a + github.com/coinbase/rosetta-sdk-go v0.4.3-0.20200914180457-7d3458f14c0f github.com/fatih/color v1.9.0 github.com/jinzhu/copier v0.0.0-20190924061706-b57f9002281a github.com/olekukonko/tablewriter v0.0.2-0.20190409134802-7e037d187b0c diff --git a/go.sum b/go.sum index b60c4668..bf2d2a81 100644 --- a/go.sum +++ b/go.sum @@ -131,6 +131,8 @@ github.com/coinbase/rosetta-sdk-go v0.4.2 h1:ojiNdDmmMwXIVIRN9vNCPytXtRBAm+EGCib github.com/coinbase/rosetta-sdk-go v0.4.2/go.mod h1:Luv0AhzZH81eul2hYZ3w0hBGwmFPiexwbntYxihEZck= github.com/coinbase/rosetta-sdk-go v0.4.3-0.20200914173330-69c73d3ff87a h1:lEI5Uy/qwhjzg+DCWJG94I+k3J+iG/zs6xjowmdzBvE= github.com/coinbase/rosetta-sdk-go v0.4.3-0.20200914173330-69c73d3ff87a/go.mod h1:Luv0AhzZH81eul2hYZ3w0hBGwmFPiexwbntYxihEZck= +github.com/coinbase/rosetta-sdk-go v0.4.3-0.20200914180457-7d3458f14c0f h1:n6zV0BCXcVYilmKw8sNiRe5rc7K6R7Gb+vX25ZIvBsM= +github.com/coinbase/rosetta-sdk-go v0.4.3-0.20200914180457-7d3458f14c0f/go.mod h1:Luv0AhzZH81eul2hYZ3w0hBGwmFPiexwbntYxihEZck= github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk= From 3334043abc9e0571ac777c470656efbd2e6d207d Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Tue, 15 Sep 2020 15:25:41 -0700 Subject: [PATCH 022/376] Support max retries --- configuration/configuration.go | 12 ++++++++---- configuration/configuration_test.go | 2 +- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/configuration/configuration.go b/configuration/configuration.go index 778f9272..6598b2df 100644 --- a/configuration/configuration.go +++ b/configuration/configuration.go @@ -54,7 +54,7 @@ const ( const ( DefaultURL = "http://localhost:8080" DefaultTimeout = 10 - DefaultRetryElapsedTime = 60 + DefaultMaxRetries = 5 DefaultMaxOnlineConnections = 120 // most OS have a default limit of 128 DefaultMaxOfflineConnections = 4 // we shouldn't need many connections for construction DefaultMaxSyncConcurrency = 64 @@ -159,7 +159,7 @@ func DefaultConfiguration() *Configuration { OnlineURL: DefaultURL, MaxOnlineConnections: DefaultMaxOnlineConnections, HTTPTimeout: DefaultTimeout, - RetryElapsedTime: DefaultRetryElapsedTime, + MaxRetries: DefaultMaxRetries, MaxSyncConcurrency: DefaultMaxSyncConcurrency, TipDelay: DefaultTipDelay, Data: DefaultDataConfiguration(), @@ -293,6 +293,10 @@ type Configuration struct { // HTTPTimeout is the timeout for a HTTP request in seconds. HTTPTimeout uint64 `json:"http_timeout"` + // MaxRetries is the number of times we will retry an HTTP request. If retry_elapsed_time + // is also populated, we may stop attempting retries early. + MaxRetries uint64 `json:"max_retries"` + // RetryElapsedTime is the total time to spend retrying a HTTP request in seconds. RetryElapsedTime uint64 `json:"retry_elapsed_time"` @@ -384,8 +388,8 @@ func populateMissingFields(config *Configuration) *Configuration { config.HTTPTimeout = DefaultTimeout } - if config.RetryElapsedTime == 0 { - config.RetryElapsedTime = DefaultRetryElapsedTime + if config.MaxRetries == 0 { + config.MaxRetries = DefaultMaxRetries } if config.MaxOnlineConnections == 0 { diff --git a/configuration/configuration_test.go b/configuration/configuration_test.go index 2994a869..30ba5818 100644 --- a/configuration/configuration_test.go +++ b/configuration/configuration_test.go @@ -51,7 +51,7 @@ var ( OnlineURL: "http://hasudhasjkdk", MaxOnlineConnections: 10, HTTPTimeout: 21, - RetryElapsedTime: 1000, + MaxRetries: 1000, MaxSyncConcurrency: 12, TipDelay: 1231, Construction: &ConstructionConfiguration{ From b98c14ab59ceb1f3de2b9023a7793d034b3a075f Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Tue, 15 Sep 2020 15:33:21 -0700 Subject: [PATCH 023/376] Ensure max retries is populated --- cmd/check_construction.go | 1 + cmd/check_data.go | 1 + cmd/utils_asserter_configuration.go | 1 + cmd/view_account.go | 1 + cmd/view_block.go | 1 + cmd/view_networks.go | 1 + pkg/tester/construction.go | 1 + 7 files changed, 7 insertions(+) diff --git a/cmd/check_construction.go b/cmd/check_construction.go index 01378c57..4b42d189 100644 --- a/cmd/check_construction.go +++ b/cmd/check_construction.go @@ -65,6 +65,7 @@ func runCheckConstructionCmd(cmd *cobra.Command, args []string) { fetcher.WithMaxConnections(Config.MaxOnlineConnections), fetcher.WithRetryElapsedTime(time.Duration(Config.RetryElapsedTime)*time.Second), fetcher.WithTimeout(time.Duration(Config.HTTPTimeout)*time.Second), + fetcher.WithMaxRetries(Config.MaxRetries), ) _, _, fetchErr := fetcher.InitializeAsserter(ctx, Config.Network) diff --git a/cmd/check_data.go b/cmd/check_data.go index 53c92585..08cefb6b 100644 --- a/cmd/check_data.go +++ b/cmd/check_data.go @@ -78,6 +78,7 @@ func runCheckDataCmd(cmd *cobra.Command, args []string) { fetcher.WithMaxConnections(Config.MaxOnlineConnections), fetcher.WithRetryElapsedTime(time.Duration(Config.RetryElapsedTime)*time.Second), fetcher.WithTimeout(time.Duration(Config.HTTPTimeout)*time.Second), + fetcher.WithMaxRetries(Config.MaxRetries), ) _, _, fetchErr := fetcher.InitializeAsserter(ctx, Config.Network) diff --git a/cmd/utils_asserter_configuration.go b/cmd/utils_asserter_configuration.go index 8b22c056..d92e998f 100644 --- a/cmd/utils_asserter_configuration.go +++ b/cmd/utils_asserter_configuration.go @@ -49,6 +49,7 @@ func runCreateConfigurationCmd(cmd *cobra.Command, args []string) { Config.OnlineURL, fetcher.WithRetryElapsedTime(time.Duration(Config.RetryElapsedTime)*time.Second), fetcher.WithTimeout(time.Duration(Config.HTTPTimeout)*time.Second), + fetcher.WithMaxRetries(Config.MaxRetries), ) // Initialize the fetcher's asserter diff --git a/cmd/view_account.go b/cmd/view_account.go index fb50fa42..55e1da24 100644 --- a/cmd/view_account.go +++ b/cmd/view_account.go @@ -63,6 +63,7 @@ func runViewAccountCmd(cmd *cobra.Command, args []string) { Config.OnlineURL, fetcher.WithRetryElapsedTime(time.Duration(Config.RetryElapsedTime)*time.Second), fetcher.WithTimeout(time.Duration(Config.HTTPTimeout)*time.Second), + fetcher.WithMaxRetries(Config.MaxRetries), ) // Initialize the fetcher's asserter diff --git a/cmd/view_block.go b/cmd/view_block.go index b0287c32..77f72ddd 100644 --- a/cmd/view_block.go +++ b/cmd/view_block.go @@ -58,6 +58,7 @@ func runViewBlockCmd(cmd *cobra.Command, args []string) { Config.OnlineURL, fetcher.WithRetryElapsedTime(time.Duration(Config.RetryElapsedTime)*time.Second), fetcher.WithTimeout(time.Duration(Config.HTTPTimeout)*time.Second), + fetcher.WithMaxRetries(Config.MaxRetries), ) // Initialize the fetcher's asserter diff --git a/cmd/view_networks.go b/cmd/view_networks.go index 74e2d549..c11c4ffb 100644 --- a/cmd/view_networks.go +++ b/cmd/view_networks.go @@ -46,6 +46,7 @@ func runViewNetworksCmd(cmd *cobra.Command, args []string) { Config.OnlineURL, fetcher.WithRetryElapsedTime(time.Duration(Config.RetryElapsedTime)*time.Second), fetcher.WithTimeout(time.Duration(Config.HTTPTimeout)*time.Second), + fetcher.WithMaxRetries(Config.MaxRetries), ) // Attempt to fetch network list diff --git a/pkg/tester/construction.go b/pkg/tester/construction.go index 0bb251dd..dd8f0a48 100644 --- a/pkg/tester/construction.go +++ b/pkg/tester/construction.go @@ -136,6 +136,7 @@ func InitializeConstruction( fetcher.WithMaxConnections(config.Construction.MaxOfflineConnections), fetcher.WithAsserter(onlineFetcher.Asserter), fetcher.WithTimeout(time.Duration(config.HTTPTimeout)*time.Second), + fetcher.WithMaxRetries(config.MaxRetries), ) // Import prefunded account and save to database From 3230f5613daf382863e3e01c65f50d0c637c8102 Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Tue, 15 Sep 2020 15:37:12 -0700 Subject: [PATCH 024/376] Update examples --- examples/configuration/default.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/examples/configuration/default.json b/examples/configuration/default.json index 7c5c722d..e6727ca9 100644 --- a/examples/configuration/default.json +++ b/examples/configuration/default.json @@ -6,7 +6,8 @@ "online_url": "http://localhost:8080", "data_directory": "", "http_timeout": 10, - "retry_elapsed_time": 60, + "max_retries": 5, + "retry_elapsed_time": 0, "max_online_connections": 120, "max_sync_concurrency": 64, "tip_delay": 300, From 319eafa7fd50857f37ca1b8ba4d9e61283a545b4 Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Tue, 15 Sep 2020 16:16:46 -0700 Subject: [PATCH 025/376] Invoke return funds if exit conditions are met --- go.mod | 2 +- go.sum | 2 ++ pkg/tester/construction.go | 14 +++++++++++--- 3 files changed, 14 insertions(+), 4 deletions(-) diff --git a/go.mod b/go.mod index ce5d1b7d..cce064ff 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/coinbase/rosetta-cli go 1.13 require ( - github.com/coinbase/rosetta-sdk-go v0.4.3-0.20200914180457-7d3458f14c0f + github.com/coinbase/rosetta-sdk-go v0.4.3 github.com/fatih/color v1.9.0 github.com/jinzhu/copier v0.0.0-20190924061706-b57f9002281a github.com/olekukonko/tablewriter v0.0.2-0.20190409134802-7e037d187b0c diff --git a/go.sum b/go.sum index bf2d2a81..fac6281e 100644 --- a/go.sum +++ b/go.sum @@ -133,6 +133,8 @@ github.com/coinbase/rosetta-sdk-go v0.4.3-0.20200914173330-69c73d3ff87a h1:lEI5U github.com/coinbase/rosetta-sdk-go v0.4.3-0.20200914173330-69c73d3ff87a/go.mod h1:Luv0AhzZH81eul2hYZ3w0hBGwmFPiexwbntYxihEZck= github.com/coinbase/rosetta-sdk-go v0.4.3-0.20200914180457-7d3458f14c0f h1:n6zV0BCXcVYilmKw8sNiRe5rc7K6R7Gb+vX25ZIvBsM= github.com/coinbase/rosetta-sdk-go v0.4.3-0.20200914180457-7d3458f14c0f/go.mod h1:Luv0AhzZH81eul2hYZ3w0hBGwmFPiexwbntYxihEZck= +github.com/coinbase/rosetta-sdk-go v0.4.3 h1:qj1U6h7textY92kmjtQmWYN+REup17+cDcNVL9drPwA= +github.com/coinbase/rosetta-sdk-go v0.4.3/go.mod h1:Luv0AhzZH81eul2hYZ3w0hBGwmFPiexwbntYxihEZck= github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk= diff --git a/pkg/tester/construction.go b/pkg/tester/construction.go index dd8f0a48..7750ad9a 100644 --- a/pkg/tester/construction.go +++ b/pkg/tester/construction.go @@ -390,9 +390,17 @@ func (t *ConstructionTester) HandleErr(err error) { return } - if t.reachedEndConditions { - ExitConstruction(t.config, t.counterStorage, t.jobStorage, nil, 0) + if !t.reachedEndConditions { + ExitConstruction(t.config, t.counterStorage, t.jobStorage, err, 1) } - ExitConstruction(t.config, t.counterStorage, t.jobStorage, err, 1) + // We optimistically run the ReturnFunds function on the coordinator + // and only log if it fails. If there is no ReturnFunds workflow defined, + // this will just return nil. + returnCtx := context.Background() + if err := t.coordinator.ReturnFunds(returnCtx); err != nil { + log.Printf("return funds failed: %v\n", err) + } + + ExitConstruction(t.config, t.counterStorage, t.jobStorage, nil, 0) } From b1b50c8e32e1936d9b40dd2ee7890f6987bc01ac Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Tue, 15 Sep 2020 18:00:50 -0700 Subject: [PATCH 026/376] Add returnFunds function and update ETH example --- cmd/check_construction.go | 4 +- cmd/check_data.go | 4 +- cmd/root.go | 4 +- examples/configuration/ethereum.json | 61 ++++++++++++++++++++++++++++ go.mod | 2 +- go.sum | 4 ++ pkg/tester/construction.go | 55 ++++++++++++++++++++++--- pkg/tester/data.go | 8 ++-- 8 files changed, 126 insertions(+), 16 deletions(-) diff --git a/cmd/check_construction.go b/cmd/check_construction.go index 4b42d189..09c15af7 100644 --- a/cmd/check_construction.go +++ b/cmd/check_construction.go @@ -132,8 +132,8 @@ func runCheckConstructionCmd(cmd *cobra.Command, args []string) { }) sigListeners := []context.CancelFunc{cancel} - go handleSignals(sigListeners) + go handleSignals(&sigListeners) err = g.Wait() - constructionTester.HandleErr(err) + constructionTester.HandleErr(err, &sigListeners) } diff --git a/cmd/check_data.go b/cmd/check_data.go index 08cefb6b..169d513d 100644 --- a/cmd/check_data.go +++ b/cmd/check_data.go @@ -138,7 +138,7 @@ func runCheckDataCmd(cmd *cobra.Command, args []string) { }) sigListeners := []context.CancelFunc{cancel} - go handleSignals(sigListeners) + go handleSignals(&sigListeners) err = g.Wait() @@ -148,5 +148,5 @@ func runCheckDataCmd(cmd *cobra.Command, args []string) { // HandleErr will exit if we should not attempt // to find missing operations. - dataTester.HandleErr(ctx, err, sigListeners) + dataTester.HandleErr(ctx, err, &sigListeners) } diff --git a/cmd/root.go b/cmd/root.go index 2939f840..e51ea79d 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -115,14 +115,14 @@ func ensureDataDirectoryExists() { // handleSignals handles OS signals so we can ensure we close database // correctly. We call multiple sigListeners because we // may need to cancel more than 1 context. -func handleSignals(listeners []context.CancelFunc) { +func handleSignals(listeners *[]context.CancelFunc) { sigs := make(chan os.Signal, 1) signal.Notify(sigs, syscall.SIGINT, syscall.SIGTERM) go func() { sig := <-sigs color.Red("Received signal: %s", sig) SignalReceived = true - for _, listener := range listeners { + for _, listener := range *listeners { listener() } }() diff --git a/examples/configuration/ethereum.json b/examples/configuration/ethereum.json index 7d8d01e0..0a3b0a21 100644 --- a/examples/configuration/ethereum.json +++ b/examples/configuration/ethereum.json @@ -6,6 +6,7 @@ "online_url": "", "data_directory": "ethereum-data", "http_timeout": 300, + "max_retries": 5, "retry_elapsed_time": 0, "max_online_connections": 0, "max_sync_concurrency": 0, @@ -147,6 +148,66 @@ ] } ] + }, + { + "name": "return_funds", + "concurrency": 10, + "scenarios": [ + { + "name": "transfer", + "actions": [ + { + "input": "{\"network\":\"Ropsten\", \"blockchain\":\"Ethereum\"}", + "type": "set_variable", + "output_path": "transfer.network" + }, + { + "input": "{\"symbol\":\"ETH\", \"decimals\":18}", + "type": "set_variable", + "output_path": "currency" + }, + { + "input": "\"42000000000000\"", + "type": "set_variable", + "output_path": "max_fee" + }, + { + "input": "{\"minimum_balance\":{\"value\": {{max_fee}}, \"currency\": {{currency}}}}", + "type": "find_balance", + "output_path": "sender" + }, + { + "input": "{\"operation\":\"subtraction\", \"left_value\": {{sender.balance.value}}, \"right_value\": {{max_fee}}}", + "type": "math", + "output_path": "available_amount" + }, + { + "input": "{\"return_amount\":{{available_amount}}}", + "type": "print_message" + }, + { + "input": "{\"operation\":\"subtraction\", \"left_value\": \"0\", \"right_value\":{{available_amount}}}", + "type": "math", + "output_path": "sender_amount" + }, + { + "input": "{\"address\":\"0xb41B39479a525AB69e38c701A713D98E3074252c\"}", + "type": "set_variable", + "output_path": "faucet" + }, + { + "input": "\"1\"", + "type": "set_variable", + "output_path": "transfer.confirmation_depth" + }, + { + "input": "[{\"operation_identifier\":{\"index\":0},\"type\":\"transfer\",\"account\":{{sender.account_identifier}},\"amount\":{\"value\":{{sender_amount}},\"currency\":{{currency}}}},{\"operation_identifier\":{\"index\":1},\"type\":\"transfer\",\"account\":{{faucet}},\"amount\":{\"value\":{{available_amount}},\"currency\":{{currency}}}}]", + "type": "set_variable", + "output_path": "transfer.operations" + } + ] + } + ] } ], "end_conditions": { diff --git a/go.mod b/go.mod index cce064ff..a83e1b1a 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/coinbase/rosetta-cli go 1.13 require ( - github.com/coinbase/rosetta-sdk-go v0.4.3 + github.com/coinbase/rosetta-sdk-go v0.4.4-0.20200916005026-5f676c313cfc github.com/fatih/color v1.9.0 github.com/jinzhu/copier v0.0.0-20190924061706-b57f9002281a github.com/olekukonko/tablewriter v0.0.2-0.20190409134802-7e037d187b0c diff --git a/go.sum b/go.sum index fac6281e..431f5d90 100644 --- a/go.sum +++ b/go.sum @@ -135,6 +135,10 @@ github.com/coinbase/rosetta-sdk-go v0.4.3-0.20200914180457-7d3458f14c0f h1:n6zV0 github.com/coinbase/rosetta-sdk-go v0.4.3-0.20200914180457-7d3458f14c0f/go.mod h1:Luv0AhzZH81eul2hYZ3w0hBGwmFPiexwbntYxihEZck= github.com/coinbase/rosetta-sdk-go v0.4.3 h1:qj1U6h7textY92kmjtQmWYN+REup17+cDcNVL9drPwA= github.com/coinbase/rosetta-sdk-go v0.4.3/go.mod h1:Luv0AhzZH81eul2hYZ3w0hBGwmFPiexwbntYxihEZck= +github.com/coinbase/rosetta-sdk-go v0.4.4-0.20200916003715-bc5c0dc3f67f h1:Phzxs8VyctyVzRzMzAml/nLOYpNw/uTfD2cZe8wyEqU= +github.com/coinbase/rosetta-sdk-go v0.4.4-0.20200916003715-bc5c0dc3f67f/go.mod h1:Luv0AhzZH81eul2hYZ3w0hBGwmFPiexwbntYxihEZck= +github.com/coinbase/rosetta-sdk-go v0.4.4-0.20200916005026-5f676c313cfc h1:huzflMX5sJfrDIhv1p+D9U60jr/WC8qzJJ2EVgAazmU= +github.com/coinbase/rosetta-sdk-go v0.4.4-0.20200916005026-5f676c313cfc/go.mod h1:Luv0AhzZH81eul2hYZ3w0hBGwmFPiexwbntYxihEZck= github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk= diff --git a/pkg/tester/construction.go b/pkg/tester/construction.go index 7750ad9a..20289fae 100644 --- a/pkg/tester/construction.go +++ b/pkg/tester/construction.go @@ -35,6 +35,7 @@ import ( "github.com/coinbase/rosetta-sdk-go/types" "github.com/coinbase/rosetta-sdk-go/utils" "github.com/fatih/color" + "golang.org/x/sync/errgroup" ) const ( @@ -382,8 +383,52 @@ func (t *ConstructionTester) WatchEndConditions( } } +func (t *ConstructionTester) returnFunds( + ctx context.Context, + sigListeners *[]context.CancelFunc, +) { + // To cancel all execution, need to call multiple cancel functions. + ctx, cancel := context.WithCancel(ctx) + *sigListeners = append(*sigListeners, cancel) + + var returnFundsSuccess bool + g, ctx := errgroup.WithContext(ctx) + g.Go(func() error { + return t.StartSyncer(ctx, cancel) + }) + g.Go(func() error { + return t.StartPeriodicLogger(ctx) + }) + g.Go(func() error { + err := t.coordinator.ReturnFunds(ctx) + + // If the error is nil, we need to cancel the syncer + // or we will sync forever. + if err == nil { + returnFundsSuccess = true // makes error parsing much easier + cancel() + return nil + } + + return err + }) + + err := g.Wait() + if *t.signalReceived { + color.Red("Fund return halted") + return + } + + if !returnFundsSuccess { + log.Printf("unable to return funds %v\n", err) + } +} + // HandleErr is called when `check:construction` returns an error. -func (t *ConstructionTester) HandleErr(err error) { +func (t *ConstructionTester) HandleErr( + err error, + sigListeners *[]context.CancelFunc, +) { if *t.signalReceived { color.Red("Check halted") os.Exit(1) @@ -397,10 +442,10 @@ func (t *ConstructionTester) HandleErr(err error) { // We optimistically run the ReturnFunds function on the coordinator // and only log if it fails. If there is no ReturnFunds workflow defined, // this will just return nil. - returnCtx := context.Background() - if err := t.coordinator.ReturnFunds(returnCtx); err != nil { - log.Printf("return funds failed: %v\n", err) - } + t.returnFunds( + context.Background(), + sigListeners, + ) ExitConstruction(t.config, t.counterStorage, t.jobStorage, nil, 0) } diff --git a/pkg/tester/data.go b/pkg/tester/data.go index b1b12f6b..e4420cf5 100644 --- a/pkg/tester/data.go +++ b/pkg/tester/data.go @@ -478,7 +478,7 @@ func (t *DataTester) WatchEndConditions( // HandleErr is called when `check:data` returns an error. // If historical balance lookups are enabled, HandleErr will attempt to // automatically find any missing balance-changing operations. -func (t *DataTester) HandleErr(ctx context.Context, err error, sigListeners []context.CancelFunc) { +func (t *DataTester) HandleErr(ctx context.Context, err error, sigListeners *[]context.CancelFunc) { if *t.signalReceived { color.Red("Check halted") os.Exit(1) @@ -533,12 +533,12 @@ func (t *DataTester) HandleErr(ctx context.Context, err error, sigListeners []co func (t *DataTester) FindMissingOps( ctx context.Context, originalErr error, - sigListeners []context.CancelFunc, + sigListeners *[]context.CancelFunc, ) { color.Cyan("Searching for block with missing operations...hold tight") badBlock, err := t.recursiveOpSearch( ctx, - &sigListeners, + sigListeners, t.reconcilerHandler.InactiveFailure, t.reconcilerHandler.InactiveFailureBlock.Index-InactiveFailureLookbackWindow, t.reconcilerHandler.InactiveFailureBlock.Index, @@ -679,7 +679,7 @@ func (t *DataTester) recursiveOpSearch( return nil, errors.New("Search for block with missing ops halted") } - if err == nil || err == context.Canceled { + if err == nil || errors.Is(err, context.Canceled) { newStart := startIndex - InactiveFailureLookbackWindow if newStart < t.genesisBlock.Index { newStart = t.genesisBlock.Index From f0473323ae0e92c9ea7f38807bf03b395d5a127e Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Tue, 15 Sep 2020 18:21:21 -0700 Subject: [PATCH 027/376] Add dry run on Bitcoin --- examples/configuration/bitcoin.json | 60 +++++++++++++++++++++++------ 1 file changed, 49 insertions(+), 11 deletions(-) diff --git a/examples/configuration/bitcoin.json b/examples/configuration/bitcoin.json index a077cec3..be697c38 100644 --- a/examples/configuration/bitcoin.json +++ b/examples/configuration/bitcoin.json @@ -3,16 +3,15 @@ "blockchain": "Bitcoin", "network": "Testnet3" }, - "online_url": "", "data_directory": "bitcoin-data", "http_timeout": 300, + "max_retries": 5, "retry_elapsed_time": 0, "max_online_connections": 0, "max_sync_concurrency": 0, "tip_delay": 1800, "log_configuration": false, "construction": { - "offline_url": "", "max_offline_connections": 0, "stale_depth": 0, "broadcast_limit": 0, @@ -88,12 +87,12 @@ "concurrency": 10, "scenarios": [ { - "name": "transfer", + "name": "transfer_dry_run", "actions": [ { "input": "{\"network\":\"Testnet3\", \"blockchain\":\"Bitcoin\"}", "type": "set_variable", - "output_path": "transfer.network" + "output_path": "transfer_dry_run.network" }, { "input": "{\"symbol\":\"tBTC\", \"decimals\":8}", @@ -106,17 +105,17 @@ "output_path": "dust_amount" }, { - "input": "\"600\"", + "input": "\"1200\"", "type": "set_variable", - "output_path": "fee_amount" + "output_path": "max_fee_amount" }, { - "input": "{\"operation\":\"addition\", \"left_value\": {{dust_amount}}, \"right_value\": {{fee_amount}}}", + "input": "{\"operation\":\"addition\", \"left_value\": {{dust_amount}}, \"right_value\": {{max_fee_amount}}}", "type": "math", "output_path": "send_buffer" }, { - "input": "\"1800\"", + "input": "\"2400\"", "type": "set_variable", "output_path": "reserved_amount" }, @@ -142,10 +141,10 @@ { "input": "{\"operation\":\"subtraction\", \"left_value\": {{sender.balance.value}}, \"right_value\": {{recipient_amount}}}", "type": "math", - "output_path": "change_amount" + "output_path": "total_change_amount" }, { - "input": "{\"operation\":\"subtraction\", \"left_value\": {{change_amount}}, \"right_value\": {{fee_amount}}}", + "input": "{\"operation\":\"subtraction\", \"left_value\": {{total_change_amount}}, \"right_value\": {{max_fee_amount}}}", "type": "math", "output_path": "change_amount" }, @@ -163,6 +162,45 @@ "type": "find_balance", "output_path": "recipient" }, + { + "input": "\"1\"", + "type": "set_variable", + "output_path": "transfer_dry_run.confirmation_depth" + }, + { + "input": "\"true\"", + "type": "set_variable", + "output_path": "transfer_dry_run.dry_run" + }, + { + "input": "[{\"operation_identifier\":{\"index\":0},\"type\":\"INPUT\",\"account\":{{sender.account_identifier}},\"amount\":{\"value\":{{sender_amount}},\"currency\":{{currency}}}, \"coin_change\":{\"coin_action\":\"coin_spent\", \"coin_identifier\":{{sender.coin}}}},{\"operation_identifier\":{\"index\":1},\"type\":\"OUTPUT\",\"account\":{{recipient.account_identifier}},\"amount\":{\"value\":{{recipient_amount}},\"currency\":{{currency}}}}, {\"operation_identifier\":{\"index\":2},\"type\":\"OUTPUT\",\"account\":{{sender.account_identifier}},\"amount\":{\"value\":{{change_amount}},\"currency\":{{currency}}}}]", + "type": "set_variable", + "output_path": "transfer_dry_run.operations" + }, + { + "input": "{{transfer_dry_run.operations}}", + "type": "print_message" + } + ] + }, + { + "name": "transfer", + "actions": [ + { + "input": "{\"currency\":{{currency}}, \"amounts\":{{transfer_dry_run.suggested_fee}}}", + "type": "find_currency_amount", + "output_path": "suggested_fee" + }, + { + "input": "{\"operation\":\"subtraction\", \"left_value\": {{total_change_amount}}, \"right_value\": {{suggested_fee.value}}}", + "type": "math", + "output_path": "change_amount" + }, + { + "input": "{\"network\":\"Testnet3\", \"blockchain\":\"Bitcoin\"}", + "type": "set_variable", + "output_path": "transfer.network" + }, { "input": "\"1\"", "type": "set_variable", @@ -208,4 +246,4 @@ }, "results_output_file": "" } -} \ No newline at end of file +} From f3254b708dbea283d60d85f36473aa894e2882ae Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Tue, 15 Sep 2020 21:54:56 -0700 Subject: [PATCH 028/376] Add return funds for bitcoin --- examples/configuration/bitcoin.json | 131 ++++++++++++++++++++++++++++ 1 file changed, 131 insertions(+) diff --git a/examples/configuration/bitcoin.json b/examples/configuration/bitcoin.json index be697c38..9896ec5c 100644 --- a/examples/configuration/bitcoin.json +++ b/examples/configuration/bitcoin.json @@ -196,6 +196,15 @@ "type": "math", "output_path": "change_amount" }, + { + "input": "{\"operation\":\"subtraction\", \"left_value\": {{change_amount}}, \"right_value\": {{dust_amount}}}", + "type": "math", + "output_path": "change_minus_dust" + }, + { + "input": "{{change_minus_dust}}", + "type": "assert" + }, { "input": "{\"network\":\"Testnet3\", \"blockchain\":\"Bitcoin\"}", "type": "set_variable", @@ -218,6 +227,128 @@ ] } ] + }, + { + "name": "return_funds", + "concurrency": 10, + "scenarios": [ + { + "name": "transfer_dry_run", + "actions": [ + { + "input": "{\"network\":\"Testnet3\", \"blockchain\":\"Bitcoin\"}", + "type": "set_variable", + "output_path": "transfer_dry_run.network" + }, + { + "input": "{\"symbol\":\"tBTC\", \"decimals\":8}", + "type": "set_variable", + "output_path": "currency" + }, + { + "input": "\"1200\"", + "type": "set_variable", + "output_path": "max_fee_amount" + }, + { + "input": "\"1800\"", + "type": "set_variable", + "output_path": "reserved_amount" + }, + { + "input": "{\"require_coin\":true, \"minimum_balance\":{\"value\": {{reserved_amount}}, \"currency\": {{currency}}}}", + "type": "find_balance", + "output_path": "sender" + }, + { + "input": "{\"operation\":\"subtraction\", \"left_value\": {{sender.balance.value}}, \"right_value\": {{max_fee_amount}}}", + "type": "math", + "output_path": "recipient_amount" + }, + { + "input": "{\"recipient_amount\":{{recipient_amount}}}", + "type": "print_message" + }, + { + "input": "{\"operation\":\"subtraction\", \"left_value\": \"0\", \"right_value\":{{sender.balance.value}}}", + "type": "math", + "output_path": "sender_amount" + }, + { + "input": "\"1\"", + "type": "set_variable", + "output_path": "transfer_dry_run.confirmation_depth" + }, + { + "input": "\"true\"", + "type": "set_variable", + "output_path": "transfer_dry_run.dry_run" + }, + { + "input": "{\"address\": \"mkHS9ne12qx9pS9VojpwU5xtRd4T7X7ZUt\"}", + "type": "set_variable", + "output_path": "recipient" + }, + { + "input": "[{\"operation_identifier\":{\"index\":0},\"type\":\"INPUT\",\"account\":{{sender.account_identifier}},\"amount\":{\"value\":{{sender_amount}},\"currency\":{{currency}}}, \"coin_change\":{\"coin_action\":\"coin_spent\", \"coin_identifier\":{{sender.coin}}}},{\"operation_identifier\":{\"index\":1},\"type\":\"OUTPUT\",\"account\":{{recipient}},\"amount\":{\"value\":{{recipient_amount}},\"currency\":{{currency}}}}]", + "type": "set_variable", + "output_path": "transfer_dry_run.operations" + }, + { + "input": "{{transfer_dry_run.operations}}", + "type": "print_message" + } + ] + }, + { + "name": "transfer", + "actions": [ + { + "input": "{\"currency\":{{currency}}, \"amounts\":{{transfer_dry_run.suggested_fee}}}", + "type": "find_currency_amount", + "output_path": "suggested_fee" + }, + { + "input": "{\"operation\":\"subtraction\", \"left_value\": {{sender.balance.value}}, \"right_value\": {{suggested_fee.value}}}", + "type": "math", + "output_path": "recipient_amount" + }, + { + "input": "\"600\"", + "type": "set_variable", + "output_path": "dust_amount" + }, + { + "input": "{\"operation\":\"subtraction\", \"left_value\": {{recipient_amount}}, \"right_value\": {{dust_amount}}}", + "type": "math", + "output_path": "recipient_minus_dust" + }, + { + "input": "{{recipient_minus_dust}}", + "type": "assert" + }, + { + "input": "{\"network\":\"Testnet3\", \"blockchain\":\"Bitcoin\"}", + "type": "set_variable", + "output_path": "transfer.network" + }, + { + "input": "\"1\"", + "type": "set_variable", + "output_path": "transfer.confirmation_depth" + }, + { + "input": "[{\"operation_identifier\":{\"index\":0},\"type\":\"INPUT\",\"account\":{{sender.account_identifier}},\"amount\":{\"value\":{{sender_amount}},\"currency\":{{currency}}}, \"coin_change\":{\"coin_action\":\"coin_spent\", \"coin_identifier\":{{sender.coin}}}},{\"operation_identifier\":{\"index\":1},\"type\":\"OUTPUT\",\"account\":{{recipient}},\"amount\":{\"value\":{{recipient_amount}},\"currency\":{{currency}}}}]", + "type": "set_variable", + "output_path": "transfer.operations" + }, + { + "input": "{{transfer.operations}}", + "type": "print_message" + } + ] + } + ] } ], "end_conditions": { From bbcb51ce7100a6c0c1aed3bc463ebd7d4ac8af59 Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Wed, 16 Sep 2020 08:34:16 -0700 Subject: [PATCH 029/376] Cleanup README --- README.md | 170 +++++++++++++++++++++++++++++++++++++++++++----------- 1 file changed, 135 insertions(+), 35 deletions(-) diff --git a/README.md b/README.md index b4b83f06..ab1884b4 100644 --- a/README.md +++ b/README.md @@ -80,13 +80,123 @@ Use "rosetta-cli [command] --help" for more information about a command. All `rosetta-cli` parameters are populated from a configuration file (`--configuration-file`) provided at runtime. If a configuration file is not provided, the default configuration is used. This default configuration can be viewed -[here](examples/configuration/default.json). +[here](examples/configuration/default.json). Note, there is no default +configuration for running `check:construction` as this is very network-specific. +You can view a full list of all configuration options [here](https://pkg.go.dev/github.com/coinbase/rosetta-cli/configuration). In the `examples/configuration` directory, you can find examples configuration files for running tests against a Bitcoin Rosetta implementation ([config](examples/configuration/bitcoin.json)) and an Ethereum Rosetta implementation ([config](examples/configuration/ethereum.json)). +#### Writing check:construction Tests +The new Construction API testing framework (first released in `rosetta-cli@v0.5.0`) uses +a new design pattern to allow for complex transaction construction orchestration. +You can read more about the design goals [here](https://community.rosetta-api.org/t/feedback-request-automated-construction-api-testing-improvements/146). + +##### Terminology +When first learning about a new topic, it is often useful to understand the +hierarchy of concerns. In the automated Construction API tester, this +"hierarchy" is as follows: +```text +Workflows -> Jobs + Scenarios + Actions +``` + +`Workflows` contain collections of `Scenarios` to execute. `Scenarios` are +executed atomically in database transactions (rolled back if execution fails) +and culminate in an optional broadcast. This means that a single `Workflow` +could contain multiple broadcasts (which can be useful for orchestrating +staking-related transactions that affect a single account). + +To perform a `Workflow`, we create a `Job`. This `Job` has a unique identifier +and stores state for all `Scenarios` in the `Workflow`. State is shared across +an entire `Job` so `Actions` in a `Scenario` can access the output of `Actions` +in other `Scenarios`. The syntax for accessing this shared state can be found +[here](https://github.com/tidwall/gjson/blob/master/SYNTAX.md). + +`Actions` are discrete operations that can be performed in the context of a +`Scenario`. A full list of all `Actions` that can be performed can be found +[here](https://pkg.go.dev/github.com/coinbase/rosetta-sdk-go/constructor/job#ActionType). + +If you have suggestions for more actions, please +[open an issue in `rosetta-sdk-go`](https://github.com/coinbase/rosetta-sdk-go/issues)! + +##### Workflows +To use the automated Construction API tester, you must implement 2 required `Workflows`: +* `create_account` +* `request_funds` + +Please note that `create_account` can contain a transaction broadcast if +on-chain origination is required for new accounts on your blockchain. + +If you plan to run the automated Construction API tester in CI, you may wish to +provide [`prefunded accounts`](https://pkg.go.dev/github.com/coinbase/rosetta-cli/configuration#ConstructionConfiguration) +when running the tester (otherwise you would need to manually fund generated +accounts). + +Optionally, you can also provide a `return_funds` workflow that will be invoked +when exiting `check:construction`. This can be useful in CI when you want to return +all funds to a single accout or faucet (instead of black-holing them in all the addresses +created during testing). + +##### Broadcast Invocation +If you'd like to broadcast a transaction at the end of a `Scenario`, +you must populate the following fields: +* `.network` +* `.operations` +* `.confirmation_depth` (allows for stake-related transactions to complete before marking as a success) + +Optionally, you can populate the following field: +* `.preprocess_metadata` + +Once a transaction is confirmed on-chain (after the provided +`.confirmation_depth`, it is stored by the tester at +`.transaction` for access by other `Scenarios` in the same `Job`. + +##### Dry Runs +In UTXO-based blockchains, it may be necessary to amend the `operations` stored +in `.operations` based on the `suggested_fee` returned in +`/construction/metadata`. The automated Construction API tester supports +running a "dry run" of a transaction broadcast if you set the follow field: +* `.dry_run = true` + +The suggested fee will then be stored as `.suggested_fee` for use by +other `Scenarios` in the same `Job`. You can find an example of this in the +Bitcoin [config](examples/configuration/bitcoin.json). + +*If this field is not populated or set to `false`, the transaction +will be constructed, signed, and broadcast.* + +##### Future Work +* DSL for writing `Workflows` (if anyone in the community has ideas for +this, we are all ears!) +* `Workflow` testing tool (to mock `Workflow` before running on network) +* Re-usable components (pre-defined logic that can be used in any `Workflow` - +both user-defined and provided by `rosetta-cli` + +#### End Conditions +When running the `rosetta-cli` in a CI job, it is usually desired to exit +when certain conditions are met (or before then with an exit code of 1). We +provide this functionality through the use of "end conditions" which can be +specified in your configuration file. + +##### check:data +A full list of `check:data` end conditions can be found [here](https://pkg.go.dev/github.com/coinbase/rosetta-cli/configuration#DataEndConditions). +If any end condition is satisifed, we will exit and output the +results in `results_output_file` (if it is populated). + +##### check:construction +The `check:construction` end condition is a map of +workflow:count that indicates how many of each workflow +should be performed before `check:construction` should stop. +For example, `{"create_account": 5}` indicates that 5 `create_account` +workflows should be performed before stopping. + +Unlike `check:data`, all `check:construction` end conditions +must be satisifed before the `rosetta-cli` will exit. + #### Disable Complex Checks If you are just getting started with your implementation, you may want to disable balance tracking (did any address balance go below zero?) and @@ -95,15 +205,6 @@ by the `/account/balance` endpoint?). Take a look at the [simple configuration](examples/configuration/simple.json) for an example of how to do this. -#### Future Work -In the near future, we will add support for providing complex exit conditions -(i.e. did we reach tip? did we reconcile every account?) for both -`check:construction` and `check:data` so that the `rosetta-cli` -can be integrated into a CI flow. Currently, the only way to exit with a -successful status in the `rosetta-cli` is to provide an `--end` flag -when running `check:data` (returns 0 if no errors up to a block index -are observed). - ### Commands #### version ``` @@ -385,31 +486,6 @@ Global Flags: default values. ``` -## Development -* `make deps` to install dependencies -* `make test` to run tests -* `make lint` to lint the source code (included generated code) -* `make release` to run one last check before opening a PR -* `make compile version=RELEASE_TAG` to generate binaries - -### Helper/Handler -Many of the packages use a `Helper/Handler` interface pattern to acquire -required information or to send events to some client implementation. An example -of this is in the `reconciler` package where a `Helper` is used to get -the account balance and the `Handler` is called to incidate whether the -reconciliation of an account was successful. - -### Repo Structure -``` -cmd -examples // examples of different config files -internal - logger // logic to write syncing information to stdout/files - processor // Helper/Handler implementations for reconciler, storage, and syncer - storage // persists block to temporary storage and allows for querying balances - utils // useful functions -``` - ## Correctness Checks This tool performs a variety of correctness checks using the Rosetta Server. If any correctness check fails, the CLI will exit and print out a detailed @@ -442,6 +518,30 @@ on the blockchain node without being included in an operation returned by the Rosetta Data API. Recall that all balance-changing operations should be returned by the Rosetta Data API. +## Development +* `make deps` to install dependencies +* `make test` to run tests +* `make lint` to lint the source code (included generated code) +* `make release` to run one last check before opening a PR +* `make compile version=RELEASE_TAG` to generate binaries + +### Helper/Handler +Many of the packages use a `Helper/Handler` interface pattern to acquire +required information or to send events to some client implementation. An example +of this is in the `reconciler` package where a `Helper` is used to get +the account balance and the `Handler` is called to incidate whether the +reconciliation of an account was successful. + +### Repo Structure +``` +cmd +examples // examples of different config files +pkg + logger // logic to write syncing information to stdout/files + processor // Helper/Handler implementations for reconciler, storage, and syncer + tester // test orchestrators +``` + ## License This project is available open source under the terms of the [Apache 2.0 License](https://opensource.org/licenses/Apache-2.0). From ddea08ee747ad5e1edc75c898c1b7d00d759d167 Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Wed, 16 Sep 2020 08:37:59 -0700 Subject: [PATCH 030/376] Update to rosetta-sdk-go@v0.4.4 --- go.mod | 2 +- go.sum | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index a83e1b1a..c8fe405c 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/coinbase/rosetta-cli go 1.13 require ( - github.com/coinbase/rosetta-sdk-go v0.4.4-0.20200916005026-5f676c313cfc + github.com/coinbase/rosetta-sdk-go v0.4.4 github.com/fatih/color v1.9.0 github.com/jinzhu/copier v0.0.0-20190924061706-b57f9002281a github.com/olekukonko/tablewriter v0.0.2-0.20190409134802-7e037d187b0c diff --git a/go.sum b/go.sum index 431f5d90..9b7ffe44 100644 --- a/go.sum +++ b/go.sum @@ -139,6 +139,8 @@ github.com/coinbase/rosetta-sdk-go v0.4.4-0.20200916003715-bc5c0dc3f67f h1:Phzxs github.com/coinbase/rosetta-sdk-go v0.4.4-0.20200916003715-bc5c0dc3f67f/go.mod h1:Luv0AhzZH81eul2hYZ3w0hBGwmFPiexwbntYxihEZck= github.com/coinbase/rosetta-sdk-go v0.4.4-0.20200916005026-5f676c313cfc h1:huzflMX5sJfrDIhv1p+D9U60jr/WC8qzJJ2EVgAazmU= github.com/coinbase/rosetta-sdk-go v0.4.4-0.20200916005026-5f676c313cfc/go.mod h1:Luv0AhzZH81eul2hYZ3w0hBGwmFPiexwbntYxihEZck= +github.com/coinbase/rosetta-sdk-go v0.4.4 h1:zTUS4bVlTfD4xq/o6JtsuU+g9sf3+S3Nnn2A24Ycow4= +github.com/coinbase/rosetta-sdk-go v0.4.4/go.mod h1:Luv0AhzZH81eul2hYZ3w0hBGwmFPiexwbntYxihEZck= github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk= From 7cec5060479190bd347adf4cb923e0f64cbf4641 Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Wed, 16 Sep 2020 08:42:33 -0700 Subject: [PATCH 031/376] nits --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index efd40154..9a5416cc 100644 --- a/Makefile +++ b/Makefile @@ -20,7 +20,7 @@ deps: go get ./... lint: - golangci-lint run -v \ + golangci-lint run --timeout 2m0s -v \ -E golint,misspell,gocyclo,whitespace,goconst,gocritic,gocognit,bodyclose,unconvert,lll,unparam,gomnd; format: From 60ac1e9c80b38de7827fcd81c6b61ed4509a8e6f Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Wed, 16 Sep 2020 08:53:43 -0700 Subject: [PATCH 032/376] Exit recursive op search if reach genesis --- pkg/tester/data.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/pkg/tester/data.go b/pkg/tester/data.go index e4420cf5..9227e3f7 100644 --- a/pkg/tester/data.go +++ b/pkg/tester/data.go @@ -676,10 +676,14 @@ func (t *DataTester) recursiveOpSearch( } if *t.signalReceived { - return nil, errors.New("Search for block with missing ops halted") + return nil, errors.New("search for block with missing ops halted") } if err == nil || errors.Is(err, context.Canceled) { + if startIndex <= t.genesisBlock.Index { + return nil, errors.New("unable to find missing ops") + } + newStart := startIndex - InactiveFailureLookbackWindow if newStart < t.genesisBlock.Index { newStart = t.genesisBlock.Index From 8a5624ca1aceff0937e60fc8c2002e9c0960ab1f Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Wed, 16 Sep 2020 09:42:34 -0700 Subject: [PATCH 033/376] Update version --- cmd/root.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/root.go b/cmd/root.go index e51ea79d..f95dde4e 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -132,6 +132,6 @@ var versionCmd = &cobra.Command{ Use: "version", Short: "Print rosetta-cli version", Run: func(cmd *cobra.Command, args []string) { - fmt.Println("v0.5.1") + fmt.Println("v0.5.2") }, } From 34c1d1639fb05ae288bd23826ff022401e53c5ef Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Fri, 18 Sep 2020 15:02:06 -0700 Subject: [PATCH 034/376] Skeleton for pruning --- cmd/check_data.go | 5 +++++ go.mod | 2 +- go.sum | 10 ++++++++++ pkg/tester/data.go | 9 +++++++++ 4 files changed, 25 insertions(+), 1 deletion(-) diff --git a/cmd/check_data.go b/cmd/check_data.go index 169d513d..f0a5f2da 100644 --- a/cmd/check_data.go +++ b/cmd/check_data.go @@ -133,6 +133,11 @@ func runCheckDataCmd(cmd *cobra.Command, args []string) { return dataTester.StartSyncing(ctx) }) + g.Go(func() error { + // TODO: make optional + return dataTester.StartPruning(ctx) + }) + g.Go(func() error { return dataTester.WatchEndConditions(ctx) }) diff --git a/go.mod b/go.mod index c8fe405c..10a167d6 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/coinbase/rosetta-cli go 1.13 require ( - github.com/coinbase/rosetta-sdk-go v0.4.4 + github.com/coinbase/rosetta-sdk-go v0.4.5-0.20200918215605-d46fa023df73 github.com/fatih/color v1.9.0 github.com/jinzhu/copier v0.0.0-20190924061706-b57f9002281a github.com/olekukonko/tablewriter v0.0.2-0.20190409134802-7e037d187b0c diff --git a/go.sum b/go.sum index 9b7ffe44..c4c564f6 100644 --- a/go.sum +++ b/go.sum @@ -141,6 +141,16 @@ github.com/coinbase/rosetta-sdk-go v0.4.4-0.20200916005026-5f676c313cfc h1:huzfl github.com/coinbase/rosetta-sdk-go v0.4.4-0.20200916005026-5f676c313cfc/go.mod h1:Luv0AhzZH81eul2hYZ3w0hBGwmFPiexwbntYxihEZck= github.com/coinbase/rosetta-sdk-go v0.4.4 h1:zTUS4bVlTfD4xq/o6JtsuU+g9sf3+S3Nnn2A24Ycow4= github.com/coinbase/rosetta-sdk-go v0.4.4/go.mod h1:Luv0AhzZH81eul2hYZ3w0hBGwmFPiexwbntYxihEZck= +github.com/coinbase/rosetta-sdk-go v0.4.5-0.20200918214028-4d968344e1a2 h1:SqJ4KAfZeyGiJc5GG9DD2vsOY/gsegJK9q9F8WaQRTc= +github.com/coinbase/rosetta-sdk-go v0.4.5-0.20200918214028-4d968344e1a2/go.mod h1:Luv0AhzZH81eul2hYZ3w0hBGwmFPiexwbntYxihEZck= +github.com/coinbase/rosetta-sdk-go v0.4.5-0.20200918214833-43eb6f18490b h1:BXy7lSdgwJeBRRo03QLzW4Bj5cRYGQJcjJKxRlUmicU= +github.com/coinbase/rosetta-sdk-go v0.4.5-0.20200918214833-43eb6f18490b/go.mod h1:Luv0AhzZH81eul2hYZ3w0hBGwmFPiexwbntYxihEZck= +github.com/coinbase/rosetta-sdk-go v0.4.5-0.20200918215107-9c608cf4c6da h1:4rH5i5ePaF5Q5IpX40ZO+0wUjvGY1WJoHfkYcbGcpYE= +github.com/coinbase/rosetta-sdk-go v0.4.5-0.20200918215107-9c608cf4c6da/go.mod h1:Luv0AhzZH81eul2hYZ3w0hBGwmFPiexwbntYxihEZck= +github.com/coinbase/rosetta-sdk-go v0.4.5-0.20200918215315-15f3222d97c4 h1:NNAwQM58a1eh/5vfrneTMeWG9hNrROlSj8Rwrc4TQW8= +github.com/coinbase/rosetta-sdk-go v0.4.5-0.20200918215315-15f3222d97c4/go.mod h1:Luv0AhzZH81eul2hYZ3w0hBGwmFPiexwbntYxihEZck= +github.com/coinbase/rosetta-sdk-go v0.4.5-0.20200918215605-d46fa023df73 h1:MDM59d52JqwUQFw0/Ow35IWxFQNkcTNIVO71MDRW9HM= +github.com/coinbase/rosetta-sdk-go v0.4.5-0.20200918215605-d46fa023df73/go.mod h1:Luv0AhzZH81eul2hYZ3w0hBGwmFPiexwbntYxihEZck= github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk= diff --git a/pkg/tester/data.go b/pkg/tester/data.go index 9227e3f7..5f982e4b 100644 --- a/pkg/tester/data.go +++ b/pkg/tester/data.go @@ -317,6 +317,15 @@ func (t *DataTester) StartSyncing( return t.syncer.Sync(ctx, startIndex, endIndex) } +// StartPruning attempts to prune block storage +// every 10 seconds. +func (t *DataTester) StartPruning( + ctx context.Context, +) error { + log.Println("starting pruning") + return t.syncer.Prune(ctx, statefulsyncer.DefaultPruningDepth) +} + // StartReconciler starts the reconciler if // reconciliation is enabled. func (t *DataTester) StartReconciler( From 57fa27eefa27a61de5161ab09dca22738610f53d Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Tue, 22 Sep 2020 09:12:59 -0700 Subject: [PATCH 035/376] Update rosetta-sdk-go version --- cmd/root.go | 2 +- go.mod | 2 +- go.sum | 2 ++ 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/cmd/root.go b/cmd/root.go index f95dde4e..a2ae1ce7 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -132,6 +132,6 @@ var versionCmd = &cobra.Command{ Use: "version", Short: "Print rosetta-cli version", Run: func(cmd *cobra.Command, args []string) { - fmt.Println("v0.5.2") + fmt.Println("v0.5.3") }, } diff --git a/go.mod b/go.mod index 10a167d6..c4d54a1d 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/coinbase/rosetta-cli go 1.13 require ( - github.com/coinbase/rosetta-sdk-go v0.4.5-0.20200918215605-d46fa023df73 + github.com/coinbase/rosetta-sdk-go v0.4.5 github.com/fatih/color v1.9.0 github.com/jinzhu/copier v0.0.0-20190924061706-b57f9002281a github.com/olekukonko/tablewriter v0.0.2-0.20190409134802-7e037d187b0c diff --git a/go.sum b/go.sum index c4c564f6..2d2b146f 100644 --- a/go.sum +++ b/go.sum @@ -151,6 +151,8 @@ github.com/coinbase/rosetta-sdk-go v0.4.5-0.20200918215315-15f3222d97c4 h1:NNAwQ github.com/coinbase/rosetta-sdk-go v0.4.5-0.20200918215315-15f3222d97c4/go.mod h1:Luv0AhzZH81eul2hYZ3w0hBGwmFPiexwbntYxihEZck= github.com/coinbase/rosetta-sdk-go v0.4.5-0.20200918215605-d46fa023df73 h1:MDM59d52JqwUQFw0/Ow35IWxFQNkcTNIVO71MDRW9HM= github.com/coinbase/rosetta-sdk-go v0.4.5-0.20200918215605-d46fa023df73/go.mod h1:Luv0AhzZH81eul2hYZ3w0hBGwmFPiexwbntYxihEZck= +github.com/coinbase/rosetta-sdk-go v0.4.5 h1:5Z+25mm/J1SStzSdTp5e8dlEMtCZRBZdOaTFCBfnP4Y= +github.com/coinbase/rosetta-sdk-go v0.4.5/go.mod h1:Luv0AhzZH81eul2hYZ3w0hBGwmFPiexwbntYxihEZck= github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk= From de03a5a69cd6733723b9d5f5584f2cb7e9188271 Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Tue, 22 Sep 2020 09:18:12 -0700 Subject: [PATCH 036/376] Add pruning config --- cmd/check_data.go | 5 ++++- configuration/configuration.go | 6 ++++++ examples/configuration/default.json | 3 ++- 3 files changed, 12 insertions(+), 2 deletions(-) diff --git a/cmd/check_data.go b/cmd/check_data.go index f0a5f2da..2ca0229b 100644 --- a/cmd/check_data.go +++ b/cmd/check_data.go @@ -134,7 +134,10 @@ func runCheckDataCmd(cmd *cobra.Command, args []string) { }) g.Go(func() error { - // TODO: make optional + if Config.Data.PruningDisabled { + return nil + } + return dataTester.StartPruning(ctx) }) diff --git a/configuration/configuration.go b/configuration/configuration.go index 6598b2df..37ffba9e 100644 --- a/configuration/configuration.go +++ b/configuration/configuration.go @@ -274,6 +274,12 @@ type DataConfiguration struct { // ResultsOutputFile is the absolute filepath of where to save // the results of a check:data run. ResultsOutputFile string `json:"results_output_file"` + + // PruningDisabled is a bolean that indicates storage pruning should + // not be attempted. This should really only ever be set to true if you + // wish to use `start_index` at a later point to restart from some + // previously synced block. + PruningDisabled bool `json:"pruning_disabled"` } // Configuration contains all configuration settings for running diff --git a/examples/configuration/default.json b/examples/configuration/default.json index e6727ca9..48b5f73b 100644 --- a/examples/configuration/default.json +++ b/examples/configuration/default.json @@ -29,6 +29,7 @@ "inactive_discrepency_search_disabled": false, "balance_tracking_disabled": false, "coin_tracking_disabled": false, - "results_output_file": "" + "results_output_file": "", + "pruning_disabled": false } } \ No newline at end of file From 173515e24b36ce7338bab08dac6298d773176fa2 Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Tue, 22 Sep 2020 09:20:44 -0700 Subject: [PATCH 037/376] nits --- cmd/check_data.go | 4 ---- pkg/tester/data.go | 5 ++++- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/cmd/check_data.go b/cmd/check_data.go index 2ca0229b..774ee652 100644 --- a/cmd/check_data.go +++ b/cmd/check_data.go @@ -134,10 +134,6 @@ func runCheckDataCmd(cmd *cobra.Command, args []string) { }) g.Go(func() error { - if Config.Data.PruningDisabled { - return nil - } - return dataTester.StartPruning(ctx) }) diff --git a/pkg/tester/data.go b/pkg/tester/data.go index 5f982e4b..10038017 100644 --- a/pkg/tester/data.go +++ b/pkg/tester/data.go @@ -322,7 +322,10 @@ func (t *DataTester) StartSyncing( func (t *DataTester) StartPruning( ctx context.Context, ) error { - log.Println("starting pruning") + if t.config.Data.PruningDisabled { + return nil + } + return t.syncer.Prune(ctx, statefulsyncer.DefaultPruningDepth) } From e3fd1becb93d58c7b3ae49160ff5e1211521e4d2 Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Wed, 23 Sep 2020 22:49:31 -0700 Subject: [PATCH 038/376] Update rosetta-sdk-go --- cmd/root.go | 2 +- go.mod | 2 +- go.sum | 2 ++ 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/cmd/root.go b/cmd/root.go index a2ae1ce7..c03681ec 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -132,6 +132,6 @@ var versionCmd = &cobra.Command{ Use: "version", Short: "Print rosetta-cli version", Run: func(cmd *cobra.Command, args []string) { - fmt.Println("v0.5.3") + fmt.Println("v0.5.4") }, } diff --git a/go.mod b/go.mod index c4d54a1d..796d8f2c 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/coinbase/rosetta-cli go 1.13 require ( - github.com/coinbase/rosetta-sdk-go v0.4.5 + github.com/coinbase/rosetta-sdk-go v0.4.6 github.com/fatih/color v1.9.0 github.com/jinzhu/copier v0.0.0-20190924061706-b57f9002281a github.com/olekukonko/tablewriter v0.0.2-0.20190409134802-7e037d187b0c diff --git a/go.sum b/go.sum index 2d2b146f..ff156c1c 100644 --- a/go.sum +++ b/go.sum @@ -153,6 +153,8 @@ github.com/coinbase/rosetta-sdk-go v0.4.5-0.20200918215605-d46fa023df73 h1:MDM59 github.com/coinbase/rosetta-sdk-go v0.4.5-0.20200918215605-d46fa023df73/go.mod h1:Luv0AhzZH81eul2hYZ3w0hBGwmFPiexwbntYxihEZck= github.com/coinbase/rosetta-sdk-go v0.4.5 h1:5Z+25mm/J1SStzSdTp5e8dlEMtCZRBZdOaTFCBfnP4Y= github.com/coinbase/rosetta-sdk-go v0.4.5/go.mod h1:Luv0AhzZH81eul2hYZ3w0hBGwmFPiexwbntYxihEZck= +github.com/coinbase/rosetta-sdk-go v0.4.6 h1:zX2SLmBF1oFbK0c4QCMwkwcHN9VjenFfIt0Dr8k8JGY= +github.com/coinbase/rosetta-sdk-go v0.4.6/go.mod h1:Luv0AhzZH81eul2hYZ3w0hBGwmFPiexwbntYxihEZck= github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk= From eb44e8a9602dd6b380f8742608f2b40bb885a2c0 Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Mon, 28 Sep 2020 22:25:57 -0700 Subject: [PATCH 039/376] Log processed blocks per second --- pkg/logger/logger.go | 17 ++++++++++++++++- pkg/tester/data.go | 17 ++++++++++++----- 2 files changed, 28 insertions(+), 6 deletions(-) diff --git a/pkg/logger/logger.go b/pkg/logger/logger.go index 053f536f..f4d916a5 100644 --- a/pkg/logger/logger.go +++ b/pkg/logger/logger.go @@ -34,6 +34,9 @@ import ( var _ statefulsyncer.Logger = (*Logger)(nil) const ( + // TimeElapsedCounter tracks the total time elapsed in seconds. + TimeElapsedCounter = "time_elapsed" + // blockStreamFile contains the stream of processed // blocks and whether they were added or removed. blockStreamFile = "blocks.txt" @@ -107,6 +110,17 @@ func (l *Logger) LogDataStats(ctx context.Context) error { return nil } + elapsedTime, err := l.CounterStorage.Get(ctx, TimeElapsedCounter) + if err != nil { + return fmt.Errorf("%w cannot get elapsed time", err) + } + + if elapsedTime.Sign() == 0 { // wait for at least some elapsed time + return nil + } + + blocksPerSecond := new(big.Int).Div(blocks, elapsedTime) + orphans, err := l.CounterStorage.Get(ctx, storage.OrphanCounter) if err != nil { return fmt.Errorf("%w cannot get orphan counter", err) @@ -133,8 +147,9 @@ func (l *Logger) LogDataStats(ctx context.Context) error { } statsMessage := fmt.Sprintf( - "[STATS] Blocks: %s (Orphaned: %s) Transactions: %s Operations: %s", + "[STATS] Blocks: %s (Blocks/Second: %s, Orphaned: %s) Transactions: %s Operations: %s", blocks.String(), + blocksPerSecond.String(), orphans.String(), txs.String(), ops.String(), diff --git a/pkg/tester/data.go b/pkg/tester/data.go index 10038017..8387f72d 100644 --- a/pkg/tester/data.go +++ b/pkg/tester/data.go @@ -19,6 +19,7 @@ import ( "errors" "fmt" "log" + "math/big" "os" "time" @@ -49,16 +50,15 @@ const ( // until the client halts the search or the block is found). InactiveFailureLookbackWindow = 250 + // periodicLoggingSeconds is the frequency to print stats in seconds. + periodicLoggingSeconds = 10 + // PeriodicLoggingFrequency is the frequency that stats are printed // to the terminal. - // - // TODO: make configurable - PeriodicLoggingFrequency = 10 * time.Second + PeriodicLoggingFrequency = periodicLoggingSeconds * time.Second // EndAtTipCheckInterval is the frequency that EndAtTip condition // is evaludated - // - // TODO: make configurable EndAtTipCheckInterval = 10 * time.Second ) @@ -357,6 +357,13 @@ func (t *DataTester) StartPeriodicLogger( return ctx.Err() case <-tc.C: + // Update the elapsed time in counter storage so that + // we can log metrics about the current check:data run. + _, _ = t.counterStorage.Update( + ctx, + logger.TimeElapsedCounter, + big.NewInt(periodicLoggingSeconds), + ) _ = t.logger.LogDataStats(ctx) } } From e634bb2bcc977c186e3a9b0dd3ce389c29a62d90 Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Mon, 28 Sep 2020 22:32:53 -0700 Subject: [PATCH 040/376] Change print format --- pkg/logger/logger.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/logger/logger.go b/pkg/logger/logger.go index f4d916a5..2d65bf82 100644 --- a/pkg/logger/logger.go +++ b/pkg/logger/logger.go @@ -147,10 +147,10 @@ func (l *Logger) LogDataStats(ctx context.Context) error { } statsMessage := fmt.Sprintf( - "[STATS] Blocks: %s (Blocks/Second: %s, Orphaned: %s) Transactions: %s Operations: %s", + "[STATS] Blocks: %s (Orphaned: %s, Rate: %s/sec) Transactions: %s Operations: %s", blocks.String(), - blocksPerSecond.String(), orphans.String(), + blocksPerSecond.String(), txs.String(), ops.String(), ) From 875a93226759777d9a5d8aa772e5a419c953c2ab Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Mon, 28 Sep 2020 23:06:54 -0700 Subject: [PATCH 041/376] Update rosetta-sdk-go --- go.mod | 2 +- go.sum | 17 +++++++++++++++++ 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index 796d8f2c..d30f00b9 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/coinbase/rosetta-cli go 1.13 require ( - github.com/coinbase/rosetta-sdk-go v0.4.6 + github.com/coinbase/rosetta-sdk-go v0.4.7 github.com/fatih/color v1.9.0 github.com/jinzhu/copier v0.0.0-20190924061706-b57f9002281a github.com/olekukonko/tablewriter v0.0.2-0.20190409134802-7e037d187b0c diff --git a/go.sum b/go.sum index ff156c1c..2698506e 100644 --- a/go.sum +++ b/go.sum @@ -13,6 +13,7 @@ github.com/Azure/go-autorest/autorest/mocks v0.3.0/go.mod h1:a8FDP3DYzQ4RYfVAxAN github.com/Azure/go-autorest/logger v0.1.0/go.mod h1:oExouG+K6PryycPJfVSxi/koC6LSNgds39diKLz7Vrc= github.com/Azure/go-autorest/tracing v0.5.0/go.mod h1:r/s2XiOKccPW3HrqB+W0TQzfbtp2fGCgRFtBroKn4Dk= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/DataDog/zstd v1.4.1 h1:3oxKN3wbHibqx897utPC2LTQU4J+IHWWJO+glkAkpFM= github.com/DataDog/zstd v1.4.1/go.mod h1:1jcaCB/ufaK+sKp1NBhlGmpz41jOoPQ35bpF36t7BBo= github.com/DataDog/zstd v1.4.5 h1:EndNeuB0l9syBZhut0wns3gV1hL8zX8LIu6ZiVHWLIQ= @@ -155,6 +156,8 @@ github.com/coinbase/rosetta-sdk-go v0.4.5 h1:5Z+25mm/J1SStzSdTp5e8dlEMtCZRBZdOaT github.com/coinbase/rosetta-sdk-go v0.4.5/go.mod h1:Luv0AhzZH81eul2hYZ3w0hBGwmFPiexwbntYxihEZck= github.com/coinbase/rosetta-sdk-go v0.4.6 h1:zX2SLmBF1oFbK0c4QCMwkwcHN9VjenFfIt0Dr8k8JGY= github.com/coinbase/rosetta-sdk-go v0.4.6/go.mod h1:Luv0AhzZH81eul2hYZ3w0hBGwmFPiexwbntYxihEZck= +github.com/coinbase/rosetta-sdk-go v0.4.7 h1:5KFc0CgLMkKamX++hYUFvE58a5/tCn0wSqpcTnDhRhY= +github.com/coinbase/rosetta-sdk-go v0.4.7/go.mod h1:8d4iN4VSGvLUzl+jRQlvYSLyS9TeY0QZebneWouizqU= github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk= @@ -198,6 +201,7 @@ github.com/dop251/goja v0.0.0-20200219165308-d1232e640a87/go.mod h1:Mw6PkjjMXWbT github.com/dop251/goja v0.0.0-20200721192441-a695b0cdd498/go.mod h1:Mw6PkjjMXWbTj+nnj4s3QPXq1jaT0s5pC0iFD4+BOAA= github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo= github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= +github.com/dvyukov/go-fuzz v0.0.0-20200318091601-be3528f3a813/go.mod h1:11Gm+ccJnvAhCNLlf5+cS9KjtbaD5I5zaZpFMsTHWTw= github.com/edsrzf/mmap-go v0.0.0-20160512033002-935e0e8a636c/go.mod h1:YO35OhQPt3KJa3ryjFM5Bs14WD66h8eGKpfaBNrHW5M= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= @@ -207,6 +211,8 @@ github.com/ethereum/go-ethereum v1.9.20 h1:kk/J5OIoaoz3DRrCXznz3RGi212mHHXwzXlY/ github.com/ethereum/go-ethereum v1.9.20/go.mod h1:JSSTypSMTkGZtAdAChH2wP5dZEvPGh3nUTuDpH+hNrg= github.com/ethereum/go-ethereum v1.9.21 h1:8qRlhzrItnmUGdVlBzZLI2Tb46S0RdSNjFwICo781ws= github.com/ethereum/go-ethereum v1.9.21/go.mod h1:RXAVzbGrSGmDkDnHymruTAIEjUR3E4TX0EOpaj702sI= +github.com/ethereum/go-ethereum v1.9.22 h1:/Fea9n2EWJuNJ9oahMq9luqjRBcbW7QWdThbcJl13ek= +github.com/ethereum/go-ethereum v1.9.22/go.mod h1:FQjK3ZwD8C5DYn7ukTmFee36rq1dOMESiUfXr5RUc1w= github.com/fatih/color v1.3.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/fatih/color v1.9.0 h1:8xPHl4/q1VyqGIPif1F+1V3Y3lSmrq01EabUW3CoW5s= github.com/fatih/color v1.9.0/go.mod h1:eQcE1qtQxscV5RaZvpXrrb8Drkc3/DdQ+uUYCNjL+zU= @@ -260,6 +266,7 @@ github.com/google/go-cmp v0.5.0 h1:/QaMHBdZ26BB3SSst0Iwl10Epc+xhTquomWX0oZEB6w= github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/gofuzz v1.1.1-0.20200604201612-c04b05f3adfa/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/pprof v0.0.0-20181127221834-b4f47329b966/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/gorilla/mux v1.7.4/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= @@ -480,11 +487,19 @@ golang.org/x/crypto v0.0.0-20200728195943-123391ffb6de/go.mod h1:LzIPMQfyMNhhGPh golang.org/x/crypto v0.0.0-20200820211705-5c72a883971a h1:vclmkQCjlDX5OydZ9wv8rBCcS0QyQY66Mpf/7BZbInM= golang.org/x/crypto v0.0.0-20200820211705-5c72a883971a/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20190731235908-ec7cb31e5a56/go.mod h1:JhuoJpWY28nO4Vef9tZUw9qufEGTyX1+7lmHxV5q5G4= +golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= +golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3 h1:XQyxROzUlZH+WIQwySDgnISgOivlhjIEwaQaJEJrrN0= golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= +golang.org/x/mobile v0.0.0-20200801112145-973feb4309de/go.mod h1:skQtrUTUwhdJvXM/2KKJzY8pDgNr9I/FOMqDVRPBUS4= golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= +golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= +golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= +golang.org/x/mod v0.1.1-0.20191209134235-331c550502dd/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0 h1:RM4zey1++hCTbCVQfnWeKs9/IEsaBLA8vTkd0WVtmH4= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= @@ -560,11 +575,13 @@ golang.org/x/tools v0.0.0-20181127232545-e782529d0ddd/go.mod h1:n7NCudcB/nEzxVGm golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20191024220359-3d91e92cde03 h1:4gtJXHJ9ud0q8MNSDxJsRU/WH+afypbe4Vk4zq+8qow= golang.org/x/tools v0.0.0-20191024220359-3d91e92cde03/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191130070609-6e064ea0cf2d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20200117012304-6edc0a871e69/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200626171337-aa94e735be7f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200729181040-64cdafbe085c/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20200731060945-b5fad4ed8dd6/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= From fb5486c518edf3055e2298a0bb7165faef817943 Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Mon, 28 Sep 2020 23:13:24 -0700 Subject: [PATCH 042/376] Wait to sync until reaching tip --- pkg/tester/construction.go | 38 +++++++++++++++++++++++++++++++------- 1 file changed, 31 insertions(+), 7 deletions(-) diff --git a/pkg/tester/construction.go b/pkg/tester/construction.go index 20289fae..c3c45e81 100644 --- a/pkg/tester/construction.go +++ b/pkg/tester/construction.go @@ -44,6 +44,7 @@ const ( constructionCmdName = "check-construction" endConditionsCheckInterval = 10 * time.Second + tipWaitInterval = 10 * time.Second ) // ConstructionTester coordinates the `check:construction` test. @@ -283,6 +284,31 @@ func (t *ConstructionTester) StartPeriodicLogger( } } +// waitForTip loops until the Rosetta implementation is at tip. +func (t *ConstructionTester) waitForTip(ctx context.Context) (int64, error) { + tc := time.NewTicker(tipWaitInterval) + defer tc.Stop() + + for { + select { + case <-ctx.Done(): + return -1, ctx.Err() + case <-tc.C: + status, fetchErr := t.onlineFetcher.NetworkStatusRetry(ctx, t.network, nil) + if fetchErr != nil { + return -1, fmt.Errorf("%w: unable to fetch network status", fetchErr.Err) + } + + // If a block has yet to be synced, start syncing from tip. + if utils.AtTip(t.config.TipDelay, status.CurrentBlockTimestamp) { + return status.CurrentBlockIdentifier.Index, nil + } + + log.Println("waiting for implementation to reach tip...") + } + } +} + // StartSyncer uses the tester's stateful syncer // to compute balance changes and track transactions // for confirmation on-chain. @@ -293,14 +319,12 @@ func (t *ConstructionTester) StartSyncer( startIndex := int64(-1) _, err := t.blockStorage.GetHeadBlockIdentifier(ctx) if errors.Is(err, storage.ErrHeadBlockNotFound) { - // If a block has yet to be synced, start syncing from tip. - // TODO: make configurable - status, fetchErr := t.onlineFetcher.NetworkStatusRetry(ctx, t.network, nil) - if fetchErr != nil { - return fmt.Errorf("%w: unable to fetch network status", fetchErr.Err) + // If no head block exists, ensure we are at tip before starting. Otherwise, + // we will unnecessarily sync tons of blocks before reaching any that matter. + startIndex, err = t.waitForTip(ctx) + if err != nil { + return fmt.Errorf("%w: unable to wait for tip", err) } - - startIndex = status.CurrentBlockIdentifier.Index } else if err != nil { return fmt.Errorf("%w: unable to get last block synced", err) } From f330eedbe561bf0b96e2f39bcf7dc6581cef2d7e Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Tue, 29 Sep 2020 09:43:45 -0700 Subject: [PATCH 043/376] Update to v0.4.8 --- go.mod | 2 +- go.sum | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index d30f00b9..823653ef 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/coinbase/rosetta-cli go 1.13 require ( - github.com/coinbase/rosetta-sdk-go v0.4.7 + github.com/coinbase/rosetta-sdk-go v0.4.8 github.com/fatih/color v1.9.0 github.com/jinzhu/copier v0.0.0-20190924061706-b57f9002281a github.com/olekukonko/tablewriter v0.0.2-0.20190409134802-7e037d187b0c diff --git a/go.sum b/go.sum index 2698506e..3c44d06b 100644 --- a/go.sum +++ b/go.sum @@ -158,6 +158,8 @@ github.com/coinbase/rosetta-sdk-go v0.4.6 h1:zX2SLmBF1oFbK0c4QCMwkwcHN9VjenFfIt0 github.com/coinbase/rosetta-sdk-go v0.4.6/go.mod h1:Luv0AhzZH81eul2hYZ3w0hBGwmFPiexwbntYxihEZck= github.com/coinbase/rosetta-sdk-go v0.4.7 h1:5KFc0CgLMkKamX++hYUFvE58a5/tCn0wSqpcTnDhRhY= github.com/coinbase/rosetta-sdk-go v0.4.7/go.mod h1:8d4iN4VSGvLUzl+jRQlvYSLyS9TeY0QZebneWouizqU= +github.com/coinbase/rosetta-sdk-go v0.4.8 h1:+E1TM4q1c5/x/jE9FPI1IZIbNvUWy4tRRgDPLnKzUV4= +github.com/coinbase/rosetta-sdk-go v0.4.8/go.mod h1:8d4iN4VSGvLUzl+jRQlvYSLyS9TeY0QZebneWouizqU= github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk= From 6af5313903effc342e1f09597ea06bd5964e3671 Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Tue, 29 Sep 2020 10:04:40 -0700 Subject: [PATCH 044/376] Log memory usage info --- cmd/check_construction.go | 4 ++++ cmd/check_data.go | 4 ++++ pkg/logger/logger.go | 14 ++++++++++++++ pkg/tester/general.go | 32 ++++++++++++++++++++++++++++++++ 4 files changed, 54 insertions(+) create mode 100644 pkg/tester/general.go diff --git a/cmd/check_construction.go b/cmd/check_construction.go index 09c15af7..6f4ded15 100644 --- a/cmd/check_construction.go +++ b/cmd/check_construction.go @@ -131,6 +131,10 @@ func runCheckConstructionCmd(cmd *cobra.Command, args []string) { return constructionTester.WatchEndConditions(ctx) }) + g.Go(func() error { + return tester.LogMemoryLoop(ctx) + }) + sigListeners := []context.CancelFunc{cancel} go handleSignals(&sigListeners) diff --git a/cmd/check_data.go b/cmd/check_data.go index 774ee652..2456a3f0 100644 --- a/cmd/check_data.go +++ b/cmd/check_data.go @@ -141,6 +141,10 @@ func runCheckDataCmd(cmd *cobra.Command, args []string) { return dataTester.WatchEndConditions(ctx) }) + g.Go(func() error { + return tester.LogMemoryLoop(ctx) + }) + sigListeners := []context.CancelFunc{cancel} go handleSignals(&sigListeners) diff --git a/pkg/logger/logger.go b/pkg/logger/logger.go index 2d65bf82..cd2427e5 100644 --- a/pkg/logger/logger.go +++ b/pkg/logger/logger.go @@ -227,6 +227,20 @@ func (l *Logger) LogConstructionStats(ctx context.Context, inflightTransactions return nil } +// LogMemoryStats logs memory usage information. +func LogMemoryStats(ctx context.Context) { + memUsage := utils.MonitorMemoryUsage(ctx, -1) + statsMessage := fmt.Sprintf( + "[MEMORY] Heap: %fMB Stack: %fMB System: %fMB GCs: %d", + memUsage.Heap, + memUsage.Stack, + memUsage.System, + memUsage.GarbageCollections, + ) + + color.Cyan(statsMessage) +} + // AddBlockStream writes the next processed block to the end of the // blockStreamFile output file. func (l *Logger) AddBlockStream( diff --git a/pkg/tester/general.go b/pkg/tester/general.go new file mode 100644 index 00000000..4f6d12a4 --- /dev/null +++ b/pkg/tester/general.go @@ -0,0 +1,32 @@ +package tester + +import ( + "context" + "time" + + "github.com/coinbase/rosetta-cli/pkg/logger" +) + +const ( + // MemoryLoggingFrequency is the frequency that memory + // usage stats are logged to the terminal. + MemoryLoggingFrequency = 10 * time.Second +) + +// LogMemoryLoop runs a loop that logs memory usage. +func LogMemoryLoop( + ctx context.Context, +) error { + ticker := time.NewTicker(MemoryLoggingFrequency) + defer ticker.Stop() + + for { + select { + case <-ctx.Done(): + logger.LogMemoryStats(ctx) + return ctx.Err() + case <-ticker.C: + logger.LogMemoryStats(ctx) + } + } +} From aa37ee4afccf80b32348f755694096ab01489fb0 Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Tue, 29 Sep 2020 10:31:04 -0700 Subject: [PATCH 045/376] Add progress logger --- cmd/check_data.go | 4 +++ pkg/logger/logger.go | 62 ++++++++++++++++++++++++++++++++++---------- pkg/tester/data.go | 27 ++++++++++++++++++- 3 files changed, 79 insertions(+), 14 deletions(-) diff --git a/cmd/check_data.go b/cmd/check_data.go index 2456a3f0..9f7b3744 100644 --- a/cmd/check_data.go +++ b/cmd/check_data.go @@ -145,6 +145,10 @@ func runCheckDataCmd(cmd *cobra.Command, args []string) { return tester.LogMemoryLoop(ctx) }) + g.Go(func() error { + return dataTester.StartProgressLogger(ctx) + }) + sigListeners := []context.CancelFunc{cancel} go handleSignals(&sigListeners) diff --git a/pkg/logger/logger.go b/pkg/logger/logger.go index cd2427e5..343b7b60 100644 --- a/pkg/logger/logger.go +++ b/pkg/logger/logger.go @@ -110,17 +110,6 @@ func (l *Logger) LogDataStats(ctx context.Context) error { return nil } - elapsedTime, err := l.CounterStorage.Get(ctx, TimeElapsedCounter) - if err != nil { - return fmt.Errorf("%w cannot get elapsed time", err) - } - - if elapsedTime.Sign() == 0 { // wait for at least some elapsed time - return nil - } - - blocksPerSecond := new(big.Int).Div(blocks, elapsedTime) - orphans, err := l.CounterStorage.Get(ctx, storage.OrphanCounter) if err != nil { return fmt.Errorf("%w cannot get orphan counter", err) @@ -147,10 +136,9 @@ func (l *Logger) LogDataStats(ctx context.Context) error { } statsMessage := fmt.Sprintf( - "[STATS] Blocks: %s (Orphaned: %s, Rate: %s/sec) Transactions: %s Operations: %s", + "[STATS] Blocks: %s (Orphaned: %s) Transactions: %s Operations: %s", blocks.String(), orphans.String(), - blocksPerSecond.String(), txs.String(), ops.String(), ) @@ -181,6 +169,54 @@ func (l *Logger) LogDataStats(ctx context.Context) error { return nil } +// LogTipEstimate logs information about the remaining blocks to sync. +func (l *Logger) LogTipEstimate(ctx context.Context, tipIndex int64) error { + blocks, err := l.CounterStorage.Get(ctx, storage.BlockCounter) + if err != nil { + return fmt.Errorf("%w cannot get block counter", err) + } + + if blocks.Sign() == 0 { // wait for at least 1 block to be processed + return nil + } + + orphans, err := l.CounterStorage.Get(ctx, storage.OrphanCounter) + if err != nil { + return fmt.Errorf("%w cannot get orphan counter", err) + } + + adjustedBlocks := blocks.Int64() - orphans.Int64() + if tipIndex-adjustedBlocks <= 0 { // return if no blocks to sync + return nil + } + + elapsedTime, err := l.CounterStorage.Get(ctx, TimeElapsedCounter) + if err != nil { + return fmt.Errorf("%w cannot get elapsed time", err) + } + + if elapsedTime.Sign() == 0 { // wait for at least some elapsed time + return nil + } + + blocksPerSecond := new(big.Float).Quo(new(big.Float).SetInt64(adjustedBlocks), new(big.Float).SetInt(elapsedTime)) + blocksPerSecondFloat, _ := blocksPerSecond.Float64() + blocksSynced := new(big.Float).Quo(new(big.Float).SetInt64(adjustedBlocks), new(big.Float).SetInt64(tipIndex)) + blocksSyncedFloat, _ := blocksSynced.Float64() + + statsMessage := fmt.Sprintf( + "[PROGRESS] Blocks Synced: %d/%d (Completed: %f%%, Rate: %f/second) Time Remaining: %s", + adjustedBlocks, + tipIndex, + blocksSyncedFloat*utils.OneHundred, + blocksPerSecondFloat, + utils.TimeToTip(blocksPerSecondFloat, adjustedBlocks, tipIndex), + ) + + color.Cyan(statsMessage) + return nil +} + // LogConstructionStats logs all construction values in CounterStorage. func (l *Logger) LogConstructionStats(ctx context.Context, inflightTransactions int) error { transactionsCreated, err := l.CounterStorage.Get(ctx, storage.TransactionsCreatedCounter) diff --git a/pkg/tester/data.go b/pkg/tester/data.go index 8387f72d..1eab0aa9 100644 --- a/pkg/tester/data.go +++ b/pkg/tester/data.go @@ -355,6 +355,24 @@ func (t *DataTester) StartPeriodicLogger( // Print stats one last time before exiting _ = t.logger.LogDataStats(ctx) + return ctx.Err() + case <-tc.C: + _ = t.logger.LogDataStats(ctx) + } + } +} + +// StartProgressLogger priunts out periodic +// estimates of sync duration if we are behind tip. +func (t *DataTester) StartProgressLogger( + ctx context.Context, +) error { + tc := time.NewTicker(PeriodicLoggingFrequency) + defer tc.Stop() + + for { + select { + case <-ctx.Done(): return ctx.Err() case <-tc.C: // Update the elapsed time in counter storage so that @@ -364,7 +382,14 @@ func (t *DataTester) StartPeriodicLogger( logger.TimeElapsedCounter, big.NewInt(periodicLoggingSeconds), ) - _ = t.logger.LogDataStats(ctx) + + status, fetchErr := t.fetcher.NetworkStatusRetry(ctx, t.network, nil) + if fetchErr != nil { + log.Printf("%v: unable to get network status\n", fetchErr.Err) + continue + } + + _ = t.logger.LogTipEstimate(ctx, status.CurrentBlockIdentifier.Index) } } } From 256234213f8e7a49df116476a3e96d7a260dbecd Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Tue, 29 Sep 2020 10:57:47 -0700 Subject: [PATCH 046/376] Don't wait for first tick --- pkg/tester/construction.go | 36 ++++++++++++++++++++++++------ pkg/tester/construction_results.go | 6 +++-- pkg/tester/data_results.go | 6 +++-- 3 files changed, 37 insertions(+), 11 deletions(-) diff --git a/pkg/tester/construction.go b/pkg/tester/construction.go index c3c45e81..da6059b3 100644 --- a/pkg/tester/construction.go +++ b/pkg/tester/construction.go @@ -284,8 +284,31 @@ func (t *ConstructionTester) StartPeriodicLogger( } } +func (t *ConstructionTester) checkTip(ctx context.Context) (int64, error) { + status, fetchErr := t.onlineFetcher.NetworkStatusRetry(ctx, t.network, nil) + if fetchErr != nil { + return -1, fmt.Errorf("%w: unable to fetch network status", fetchErr.Err) + } + + // If a block has yet to be synced, start syncing from tip. + if utils.AtTip(t.config.TipDelay, status.CurrentBlockTimestamp) { + return status.CurrentBlockIdentifier.Index, nil + } + + return -1, nil +} + // waitForTip loops until the Rosetta implementation is at tip. func (t *ConstructionTester) waitForTip(ctx context.Context) (int64, error) { + // Don't wait any time before first tick if at tip. + tipIndex, err := t.checkTip(ctx) + if err != nil { + return -1, err + } + if tipIndex != -1 { + return tipIndex, nil + } + tc := time.NewTicker(tipWaitInterval) defer tc.Stop() @@ -294,17 +317,16 @@ func (t *ConstructionTester) waitForTip(ctx context.Context) (int64, error) { case <-ctx.Done(): return -1, ctx.Err() case <-tc.C: - status, fetchErr := t.onlineFetcher.NetworkStatusRetry(ctx, t.network, nil) - if fetchErr != nil { - return -1, fmt.Errorf("%w: unable to fetch network status", fetchErr.Err) + tipIndex, err := t.checkTip(ctx) + if err != nil { + return -1, err } - // If a block has yet to be synced, start syncing from tip. - if utils.AtTip(t.config.TipDelay, status.CurrentBlockTimestamp) { - return status.CurrentBlockIdentifier.Index, nil + if tipIndex != -1 { + return tipIndex, nil } - log.Println("waiting for implementation to reach tip...") + log.Println("waiting for implementation to reach tip before testing...") } } } diff --git a/pkg/tester/construction_results.go b/pkg/tester/construction_results.go index 55ff54b0..00b26c77 100644 --- a/pkg/tester/construction_results.go +++ b/pkg/tester/construction_results.go @@ -240,8 +240,10 @@ func ExitConstruction( counterStorage, jobStorage, ) - results.Print() - results.Output(config.Construction.ResultsOutputFile) + if results != nil { + results.Print() + results.Output(config.Construction.ResultsOutputFile) + } os.Exit(status) } diff --git a/pkg/tester/data_results.go b/pkg/tester/data_results.go index 06252dad..09b8d4ce 100644 --- a/pkg/tester/data_results.go +++ b/pkg/tester/data_results.go @@ -471,8 +471,10 @@ func ExitData( endCondition, endConditionDetail, ) - results.Print() - results.Output(config.Data.ResultsOutputFile) + if results != nil { + results.Print() + results.Output(config.Data.ResultsOutputFile) + } os.Exit(status) } From 0e71795a04be739fdd0a2098f0fc7a6a59461032 Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Tue, 29 Sep 2020 10:58:23 -0700 Subject: [PATCH 047/376] Update version --- cmd/root.go | 2 +- pkg/tester/construction.go | 32 +++++++++++++------------------- 2 files changed, 14 insertions(+), 20 deletions(-) diff --git a/cmd/root.go b/cmd/root.go index c03681ec..33c1fd18 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -132,6 +132,6 @@ var versionCmd = &cobra.Command{ Use: "version", Short: "Print rosetta-cli version", Run: func(cmd *cobra.Command, args []string) { - fmt.Println("v0.5.4") + fmt.Println("v0.5.5") }, } diff --git a/pkg/tester/construction.go b/pkg/tester/construction.go index da6059b3..75ac867a 100644 --- a/pkg/tester/construction.go +++ b/pkg/tester/construction.go @@ -300,33 +300,27 @@ func (t *ConstructionTester) checkTip(ctx context.Context) (int64, error) { // waitForTip loops until the Rosetta implementation is at tip. func (t *ConstructionTester) waitForTip(ctx context.Context) (int64, error) { - // Don't wait any time before first tick if at tip. - tipIndex, err := t.checkTip(ctx) - if err != nil { - return -1, err - } - if tipIndex != -1 { - return tipIndex, nil - } - tc := time.NewTicker(tipWaitInterval) defer tc.Stop() for { + // Don't wait any time before first tick if at tip. + tipIndex, err := t.checkTip(ctx) + if err != nil { + return -1, err + } + + if tipIndex != -1 { + return tipIndex, nil + } + + log.Println("waiting for implementation to reach tip before testing...") + select { case <-ctx.Done(): return -1, ctx.Err() case <-tc.C: - tipIndex, err := t.checkTip(ctx) - if err != nil { - return -1, err - } - - if tipIndex != -1 { - return tipIndex, nil - } - - log.Println("waiting for implementation to reach tip before testing...") + continue } } } From 0babed08d3b4902c688a359d92e59fa9183e337f Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Tue, 29 Sep 2020 11:06:11 -0700 Subject: [PATCH 048/376] Add license --- go.sum | 1 + pkg/tester/general.go | 14 ++++++++++++++ 2 files changed, 15 insertions(+) diff --git a/go.sum b/go.sum index 3c44d06b..a2849f95 100644 --- a/go.sum +++ b/go.sum @@ -257,6 +257,7 @@ github.com/golang/snappy v0.0.2-0.20200707131729-196ae77b8a26 h1:lMm2hD9Fy0ynom5 github.com/golang/snappy v0.0.2-0.20200707131729-196ae77b8a26/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/google/addlicense v0.0.0-20200817051935-6f4cd4aacc89 h1:oTppmscIAQ2Y1tcsMDcTLR3z4MN/96/pvIsBSLGl7o8= github.com/google/addlicense v0.0.0-20200817051935-6f4cd4aacc89/go.mod h1:EMjYTRimagHs1FwlIqKyX3wAM0u3rA+McvlIIWmSamA= +github.com/google/addlicense v0.0.0-20200827091314-d1655b921368 h1:Ds6gDZHoviaQM7r7oMx/cG2qwZc3l5u7cg6gTkxOZNE= github.com/google/addlicense v0.0.0-20200827091314-d1655b921368/go.mod h1:EMjYTRimagHs1FwlIqKyX3wAM0u3rA+McvlIIWmSamA= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= diff --git a/pkg/tester/general.go b/pkg/tester/general.go index 4f6d12a4..ae636ca1 100644 --- a/pkg/tester/general.go +++ b/pkg/tester/general.go @@ -1,3 +1,17 @@ +// Copyright 2020 Coinbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package tester import ( From bf1892bdee36bcd071b30617e4c0a765a48d2333 Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Tue, 29 Sep 2020 15:28:27 -0700 Subject: [PATCH 049/376] Start work on status endpoint data --- configuration/configuration.go | 10 ++++++++++ pkg/tester/data.go | 17 +++++++++++++++++ 2 files changed, 27 insertions(+) diff --git a/configuration/configuration.go b/configuration/configuration.go index 37ffba9e..ce71cf6e 100644 --- a/configuration/configuration.go +++ b/configuration/configuration.go @@ -135,6 +135,11 @@ type ConstructionConfiguration struct { // workflows should be performed before stopping. EndConditions map[string]int `json:"end_conditions,omitempty"` + // StatusPort allows the caller to query a running check:construction + // test to get stats about progress. This can be used instead + // of parsing logs to populate some sort of status dashboard. + StatusPort uint `json:"status_port,omitempty"` + // ResultsOutputFile is the absolute filepath of where to save // the results of a check:construction run. ResultsOutputFile string `json:"results_output_file,omitempty"` @@ -271,6 +276,11 @@ type DataConfiguration struct { // EndCondition contains the conditions for the syncer to stop EndConditions *DataEndConditions `json:"end_conditions,omitempty"` + // StatusPort allows the caller to query a running check:data + // test to get stats about progress. This can be used instead + // of parsing logs to populate some sort of status dashboard. + StatusPort uint `json:"status_port,omitempty"` + // ResultsOutputFile is the absolute filepath of where to save // the results of a check:data run. ResultsOutputFile string `json:"results_output_file"` diff --git a/pkg/tester/data.go b/pkg/tester/data.go index 1eab0aa9..8ed69344 100644 --- a/pkg/tester/data.go +++ b/pkg/tester/data.go @@ -62,6 +62,23 @@ const ( EndAtTipCheckInterval = 10 * time.Second ) +type ConstructionStatus struct { + Stats *CheckConstructionStats `json:"stats"` +} + +type CheckDataProgress struct { + Blocks int64 `json:"blocks"` + Tip int64 `json:"tip"` + Completed float64 `json:"completed"` + Rate float64 `json:"rate"` + TimeRemaining string `json:"time_remaining"` +} + +type DataStatus struct { + Stats *CheckDataStats `json:"stats"` + Progress *CheckDataProgress `json:"progress"` +} + // DataTester coordinates the `check:data` test. type DataTester struct { network *types.NetworkIdentifier From 4a3502d050a9c93f5a782474dba20dfd0cd608e7 Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Tue, 29 Sep 2020 15:42:31 -0700 Subject: [PATCH 050/376] Set default status port --- configuration/configuration.go | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/configuration/configuration.go b/configuration/configuration.go index ce71cf6e..4fc5981b 100644 --- a/configuration/configuration.go +++ b/configuration/configuration.go @@ -66,6 +66,7 @@ const ( DefaultBroadcastLimit = 3 DefaultTipDelay = 300 DefaultBlockBroadcastLimit = 5 + DefaultStatusPort = 9090 // ETH Defaults EthereumIDBlockchain = "Ethereum" @@ -152,6 +153,7 @@ func DefaultDataConfiguration() *DataConfiguration { ActiveReconciliationConcurrency: DefaultActiveReconciliationConcurrency, InactiveReconciliationConcurrency: DefaultInactiveReconciliationConcurrency, InactiveReconciliationFrequency: DefaultInactiveReconciliationFrequency, + StatusPort: DefaultStatusPort, } } @@ -364,6 +366,10 @@ func populateConstructionMissingFields( constructionConfig.BlockBroadcastLimit = DefaultBlockBroadcastLimit } + if constructionConfig.StatusPort == 0 { + constructionConfig.StatusPort = DefaultStatusPort + } + return constructionConfig } @@ -384,6 +390,10 @@ func populateDataMissingFields(dataConfig *DataConfiguration) *DataConfiguration dataConfig.InactiveReconciliationFrequency = DefaultInactiveReconciliationFrequency } + if dataConfig.StatusPort == 0 { + dataConfig.StatusPort = DefaultStatusPort + } + return dataConfig } From e210c5a29b8fd4a91ec83b7b0783792fef5b4eeb Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Tue, 29 Sep 2020 17:20:30 -0700 Subject: [PATCH 051/376] Ugly first pass at status endpoint --- cmd/check_data.go | 4 +++ pkg/tester/data.go | 67 +++++++++++++++++++++++++++++++++----- pkg/tester/data_results.go | 60 ++++++++++++++++++++++++++++++++++ 3 files changed, 123 insertions(+), 8 deletions(-) diff --git a/cmd/check_data.go b/cmd/check_data.go index 9f7b3744..93070bb6 100644 --- a/cmd/check_data.go +++ b/cmd/check_data.go @@ -149,6 +149,10 @@ func runCheckDataCmd(cmd *cobra.Command, args []string) { return dataTester.StartProgressLogger(ctx) }) + g.Go(func() error { + return dataTester.StartStatusEndpoint(ctx) + }) + sigListeners := []context.CancelFunc{cancel} go handleSignals(&sigListeners) diff --git a/pkg/tester/data.go b/pkg/tester/data.go index 8ed69344..0b7a7121 100644 --- a/pkg/tester/data.go +++ b/pkg/tester/data.go @@ -16,10 +16,12 @@ package tester import ( "context" + "encoding/json" "errors" "fmt" "log" "math/big" + "net/http" "os" "time" @@ -66,14 +68,6 @@ type ConstructionStatus struct { Stats *CheckConstructionStats `json:"stats"` } -type CheckDataProgress struct { - Blocks int64 `json:"blocks"` - Tip int64 `json:"tip"` - Completed float64 `json:"completed"` - Rate float64 `json:"rate"` - TimeRemaining string `json:"time_remaining"` -} - type DataStatus struct { Stats *CheckDataStats `json:"stats"` Progress *CheckDataProgress `json:"progress"` @@ -411,6 +405,63 @@ func (t *DataTester) StartProgressLogger( } } +type StatusHandler struct { + Tester *DataTester +} + +func (h *StatusHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json; charset=UTF-8") + w.WriteHeader(http.StatusOK) + + networkStatus, fetchErr := h.Tester.fetcher.NetworkStatusRetry(r.Context(), h.Tester.network, nil) + if fetchErr != nil { + http.Error(w, fetchErr.Err.Error(), http.StatusInternalServerError) + return + } + + status := &DataStatus{ + Stats: ComputeCheckDataStats( + r.Context(), + h.Tester.counterStorage, + h.Tester.balanceStorage, + ), + Progress: ComputeCheckDataProgress( + r.Context(), + networkStatus.CurrentBlockIdentifier.Index, + h.Tester.counterStorage, + ), + } + + if err := json.NewEncoder(w).Encode(status); err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + } +} + +func (t *DataTester) StartStatusEndpoint( + ctx context.Context, +) error { + server := &http.Server{ + Addr: fmt.Sprintf(":%d", t.config.Data.StatusPort), + Handler: &StatusHandler{Tester: t}, + } + + go func() { + log.Printf("status server running on port %d\n", t.config.Data.StatusPort) + server.ListenAndServe() + }() + + go func() { + // If we don't shutdown server, it will + // never stop because server.ListenAndServe doesn't + // take any context. + <-ctx.Done() + + server.Shutdown(ctx) + }() + + return ctx.Err() +} + // EndAtTipLoop runs a loop that evaluates end condition EndAtTip func (t *DataTester) EndAtTipLoop( ctx context.Context, diff --git a/pkg/tester/data_results.go b/pkg/tester/data_results.go index 09b8d4ce..ec7ad378 100644 --- a/pkg/tester/data_results.go +++ b/pkg/tester/data_results.go @@ -19,10 +19,12 @@ import ( "errors" "fmt" "log" + "math/big" "os" "strconv" "github.com/coinbase/rosetta-cli/configuration" + "github.com/coinbase/rosetta-cli/pkg/logger" "github.com/coinbase/rosetta-cli/pkg/processor" "github.com/coinbase/rosetta-sdk-go/asserter" @@ -209,6 +211,64 @@ func ComputeCheckDataStats( return stats } +type CheckDataProgress struct { + Blocks int64 `json:"blocks"` + Tip int64 `json:"tip"` + Completed float64 `json:"completed"` + Rate float64 `json:"rate"` + TimeRemaining string `json:"time_remaining"` +} + +func ComputeCheckDataProgress( + ctx context.Context, + tipIndex int64, + counters *storage.CounterStorage, +) *CheckDataProgress { + blocks, err := counters.Get(ctx, storage.BlockCounter) + if err != nil { + fmt.Printf("%s: cannot get block counter", err.Error()) + return nil + } + + if blocks.Sign() == 0 { // wait for at least 1 block to be processed + return nil + } + + orphans, err := counters.Get(ctx, storage.OrphanCounter) + if err != nil { + fmt.Printf("%s: cannot get orphan counter", err.Error()) + return nil + } + + adjustedBlocks := blocks.Int64() - orphans.Int64() + if tipIndex-adjustedBlocks <= 0 { // return if no blocks to sync + return nil + } + + elapsedTime, err := counters.Get(ctx, logger.TimeElapsedCounter) + if err != nil { + fmt.Printf("%s: cannot get elapsed time", err.Error()) + return nil + } + + if elapsedTime.Sign() == 0 { // wait for at least some elapsed time + return nil + } + + blocksPerSecond := new(big.Float).Quo(new(big.Float).SetInt64(adjustedBlocks), new(big.Float).SetInt(elapsedTime)) + blocksPerSecondFloat, _ := blocksPerSecond.Float64() + blocksSynced := new(big.Float).Quo(new(big.Float).SetInt64(adjustedBlocks), new(big.Float).SetInt64(tipIndex)) + blocksSyncedFloat, _ := blocksSynced.Float64() + + return &CheckDataProgress{ + Blocks: adjustedBlocks, + Tip: tipIndex, + Completed: blocksSyncedFloat * utils.OneHundred, + Rate: blocksPerSecondFloat, + TimeRemaining: utils.TimeToTip(blocksPerSecondFloat, adjustedBlocks, tipIndex).String(), + } +} + // CheckDataTests indicates which tests passed. // If a test is nil, it did not apply to the run. // From 7d0c5613eddb19ed8edc04a028860d2d472fb303 Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Tue, 29 Sep 2020 17:52:03 -0700 Subject: [PATCH 052/376] First pass at starting cleanup --- cmd/check_data.go | 7 ++++- pkg/tester/data.go | 62 ++++++-------------------------------- pkg/tester/data_results.go | 46 +++++++++++++++++++++++++++- pkg/tester/general.go | 34 +++++++++++++++++++++ 4 files changed, 94 insertions(+), 55 deletions(-) diff --git a/cmd/check_data.go b/cmd/check_data.go index 93070bb6..f25c60c3 100644 --- a/cmd/check_data.go +++ b/cmd/check_data.go @@ -150,7 +150,12 @@ func runCheckDataCmd(cmd *cobra.Command, args []string) { }) g.Go(func() error { - return dataTester.StartStatusEndpoint(ctx) + return tester.StartServer( + ctx, + "check:data status", + dataTester, + Config.Data.StatusPort, + ) }) sigListeners := []context.CancelFunc{cancel} diff --git a/pkg/tester/data.go b/pkg/tester/data.go index 0b7a7121..002ea9a5 100644 --- a/pkg/tester/data.go +++ b/pkg/tester/data.go @@ -68,11 +68,6 @@ type ConstructionStatus struct { Stats *CheckConstructionStats `json:"stats"` } -type DataStatus struct { - Stats *CheckDataStats `json:"stats"` - Progress *CheckDataProgress `json:"progress"` -} - // DataTester coordinates the `check:data` test. type DataTester struct { network *types.NetworkIdentifier @@ -405,63 +400,24 @@ func (t *DataTester) StartProgressLogger( } } -type StatusHandler struct { - Tester *DataTester -} - -func (h *StatusHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { +// ServeHTTP serves a CheckDataStatus response on all paths. +func (t *DataTester) ServeHTTP(w http.ResponseWriter, r *http.Request) { w.Header().Set("Content-Type", "application/json; charset=UTF-8") w.WriteHeader(http.StatusOK) - networkStatus, fetchErr := h.Tester.fetcher.NetworkStatusRetry(r.Context(), h.Tester.network, nil) - if fetchErr != nil { - http.Error(w, fetchErr.Err.Error(), http.StatusInternalServerError) - return - } - - status := &DataStatus{ - Stats: ComputeCheckDataStats( - r.Context(), - h.Tester.counterStorage, - h.Tester.balanceStorage, - ), - Progress: ComputeCheckDataProgress( - r.Context(), - networkStatus.CurrentBlockIdentifier.Index, - h.Tester.counterStorage, - ), - } + status := ComputeCheckDataStatus( + r.Context(), + t.counterStorage, + t.balanceStorage, + t.fetcher, + t.network, + ) if err := json.NewEncoder(w).Encode(status); err != nil { http.Error(w, err.Error(), http.StatusInternalServerError) } } -func (t *DataTester) StartStatusEndpoint( - ctx context.Context, -) error { - server := &http.Server{ - Addr: fmt.Sprintf(":%d", t.config.Data.StatusPort), - Handler: &StatusHandler{Tester: t}, - } - - go func() { - log.Printf("status server running on port %d\n", t.config.Data.StatusPort) - server.ListenAndServe() - }() - - go func() { - // If we don't shutdown server, it will - // never stop because server.ListenAndServe doesn't - // take any context. - <-ctx.Done() - - server.Shutdown(ctx) - }() - - return ctx.Err() -} - // EndAtTipLoop runs a loop that evaluates end condition EndAtTip func (t *DataTester) EndAtTipLoop( ctx context.Context, diff --git a/pkg/tester/data_results.go b/pkg/tester/data_results.go index ec7ad378..17c49713 100644 --- a/pkg/tester/data_results.go +++ b/pkg/tester/data_results.go @@ -31,6 +31,7 @@ import ( "github.com/coinbase/rosetta-sdk-go/fetcher" "github.com/coinbase/rosetta-sdk-go/storage" "github.com/coinbase/rosetta-sdk-go/syncer" + "github.com/coinbase/rosetta-sdk-go/types" "github.com/coinbase/rosetta-sdk-go/utils" "github.com/fatih/color" "github.com/olekukonko/tablewriter" @@ -211,6 +212,8 @@ func ComputeCheckDataStats( return stats } +// CheckDataProgress contains information +// about check:data's syncing progress. type CheckDataProgress struct { Blocks int64 `json:"blocks"` Tip int64 `json:"tip"` @@ -219,11 +222,21 @@ type CheckDataProgress struct { TimeRemaining string `json:"time_remaining"` } +// ComputeCheckDataProgress returns +// a populated *CheckDataProgress. func ComputeCheckDataProgress( ctx context.Context, - tipIndex int64, + fetcher *fetcher.Fetcher, + network *types.NetworkIdentifier, counters *storage.CounterStorage, ) *CheckDataProgress { + networkStatus, fetchErr := fetcher.NetworkStatusRetry(ctx, network, nil) + if fetchErr != nil { + fmt.Printf("%s: cannot get network status", fetchErr.Err.Error()) + return nil + } + tipIndex := networkStatus.CurrentBlockIdentifier.Index + blocks, err := counters.Get(ctx, storage.BlockCounter) if err != nil { fmt.Printf("%s: cannot get block counter", err.Error()) @@ -269,6 +282,37 @@ func ComputeCheckDataProgress( } } +// CheckDataStatus contains both CheckDataStats +// and CheckDataProgress. +type CheckDataStatus struct { + Stats *CheckDataStats `json:"stats"` + Progress *CheckDataProgress `json:"progress"` +} + +// ComputeCheckDataStatus returns a populated +// *CheckDataStatus. +func ComputeCheckDataStatus( + ctx context.Context, + counters *storage.CounterStorage, + balances *storage.BalanceStorage, + fetcher *fetcher.Fetcher, + network *types.NetworkIdentifier, +) *CheckDataStatus { + return &CheckDataStatus{ + Stats: ComputeCheckDataStats( + ctx, + counters, + balances, + ), + Progress: ComputeCheckDataProgress( + ctx, + fetcher, + network, + counters, + ), + } +} + // CheckDataTests indicates which tests passed. // If a test is nil, it did not apply to the run. // diff --git a/pkg/tester/general.go b/pkg/tester/general.go index ae636ca1..dbc4405d 100644 --- a/pkg/tester/general.go +++ b/pkg/tester/general.go @@ -16,6 +16,9 @@ package tester import ( "context" + "fmt" + "log" + "net/http" "time" "github.com/coinbase/rosetta-cli/pkg/logger" @@ -44,3 +47,34 @@ func LogMemoryLoop( } } } + +// StartServer stats a server at a port with a particular handler. +// This is often used to support a status endpoint for a particular test. +func StartServer( + ctx context.Context, + name string, + handler http.Handler, + port uint, +) error { + server := &http.Server{ + Addr: fmt.Sprintf(":%d", port), + Handler: handler, + } + + go func() { + log.Printf("%s server running on port %d\n", name, port) + server.ListenAndServe() + }() + + go func() { + // If we don't shutdown server, it will + // never stop because server.ListenAndServe doesn't + // take any context. + <-ctx.Done() + log.Printf("%s server shutting down", name) + + server.Shutdown(ctx) + }() + + return ctx.Err() +} From 6cfca074b03ad07f90a2548ed34ef4c79288a717 Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Tue, 29 Sep 2020 17:59:01 -0700 Subject: [PATCH 053/376] Continue cleaning up abstraction --- cmd/check_construction.go | 9 +++++++++ pkg/tester/construction.go | 19 +++++++++++++++++++ pkg/tester/construction_results.go | 18 ++++++++++++++++++ pkg/tester/data.go | 4 ---- 4 files changed, 46 insertions(+), 4 deletions(-) diff --git a/cmd/check_construction.go b/cmd/check_construction.go index 6f4ded15..e3c99afa 100644 --- a/cmd/check_construction.go +++ b/cmd/check_construction.go @@ -135,6 +135,15 @@ func runCheckConstructionCmd(cmd *cobra.Command, args []string) { return tester.LogMemoryLoop(ctx) }) + g.Go(func() error { + return tester.StartServer( + ctx, + "check:construction status", + constructionTester, + Config.Construction.StatusPort, + ) + }) + sigListeners := []context.CancelFunc{cancel} go handleSignals(&sigListeners) diff --git a/pkg/tester/construction.go b/pkg/tester/construction.go index 75ac867a..1e170b62 100644 --- a/pkg/tester/construction.go +++ b/pkg/tester/construction.go @@ -16,9 +16,11 @@ package tester import ( "context" + "encoding/json" "errors" "fmt" "log" + "net/http" "os" "time" @@ -366,6 +368,23 @@ func (t *ConstructionTester) StartConstructor( return t.coordinator.Process(ctx) } +// ServeHTTP serves a CheckDataStatus response on all paths. +func (t *ConstructionTester) ServeHTTP(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json; charset=UTF-8") + w.WriteHeader(http.StatusOK) + + status := ComputeCheckConstructionStatus( + r.Context(), + t.config, + t.counterStorage, + t.jobStorage, + ) + + if err := json.NewEncoder(w).Encode(status); err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + } +} + // PerformBroadcasts attempts to rebroadcast all pending transactions // if the RebroadcastAll configuration is set to true. func (t *ConstructionTester) PerformBroadcasts(ctx context.Context) error { diff --git a/pkg/tester/construction_results.go b/pkg/tester/construction_results.go index 00b26c77..0bbaf8c4 100644 --- a/pkg/tester/construction_results.go +++ b/pkg/tester/construction_results.go @@ -225,6 +225,24 @@ func ComputeCheckConstructionStats( } } +// CheckConstructionStatus contains CheckConstructionStats. +type CheckConstructionStatus struct { + Stats *CheckConstructionStats `json:"stats"` +} + +// ComputeCheckConstructionStatus returns a populated +// *CheckConstructionStatus. +func ComputeCheckConstructionStatus( + ctx context.Context, + config *configuration.Configuration, + counters *storage.CounterStorage, + jobs *storage.JobStorage, +) *CheckConstructionStatus { + return &CheckConstructionStatus{ + Stats: ComputeCheckConstructionStats(ctx, config, counters, jobs), + } +} + // ExitConstruction exits check:data, logs the test results to the console, // and to a provided output path. func ExitConstruction( diff --git a/pkg/tester/data.go b/pkg/tester/data.go index 002ea9a5..82c2cda0 100644 --- a/pkg/tester/data.go +++ b/pkg/tester/data.go @@ -64,10 +64,6 @@ const ( EndAtTipCheckInterval = 10 * time.Second ) -type ConstructionStatus struct { - Stats *CheckConstructionStats `json:"stats"` -} - // DataTester coordinates the `check:data` test. type DataTester struct { network *types.NetworkIdentifier From 64fec0e5cd3ace6e8621c1b8364fd9dff19deb11 Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Tue, 29 Sep 2020 21:09:32 -0700 Subject: [PATCH 054/376] Cleanup results --- cmd/check_construction.go | 7 ++++--- cmd/check_data.go | 5 +++-- pkg/{tester => results}/construction_results.go | 2 +- pkg/{tester => results}/data_results.go | 2 +- pkg/{tester => results}/data_results_test.go | 2 +- pkg/tester/construction.go | 9 ++++++--- pkg/tester/data.go | 17 ++++++++++------- pkg/tester/general.go | 4 ++-- 8 files changed, 28 insertions(+), 20 deletions(-) rename pkg/{tester => results}/construction_results.go (99%) rename pkg/{tester => results}/data_results.go (99%) rename pkg/{tester => results}/data_results_test.go (99%) diff --git a/cmd/check_construction.go b/cmd/check_construction.go index e3c99afa..05bbbddf 100644 --- a/cmd/check_construction.go +++ b/cmd/check_construction.go @@ -20,6 +20,7 @@ import ( "log" "time" + "github.com/coinbase/rosetta-cli/pkg/results" "github.com/coinbase/rosetta-cli/pkg/tester" "github.com/coinbase/rosetta-sdk-go/fetcher" @@ -70,7 +71,7 @@ func runCheckConstructionCmd(cmd *cobra.Command, args []string) { _, _, fetchErr := fetcher.InitializeAsserter(ctx, Config.Network) if fetchErr != nil { - tester.ExitConstruction( + results.ExitConstruction( Config, nil, nil, @@ -81,7 +82,7 @@ func runCheckConstructionCmd(cmd *cobra.Command, args []string) { _, err := utils.CheckNetworkSupported(ctx, Config.Network, fetcher) if err != nil { - tester.ExitConstruction( + results.ExitConstruction( Config, nil, nil, @@ -99,7 +100,7 @@ func runCheckConstructionCmd(cmd *cobra.Command, args []string) { &SignalReceived, ) if err != nil { - tester.ExitConstruction( + results.ExitConstruction( Config, nil, nil, diff --git a/cmd/check_data.go b/cmd/check_data.go index f25c60c3..2ba41654 100644 --- a/cmd/check_data.go +++ b/cmd/check_data.go @@ -19,6 +19,7 @@ import ( "fmt" "time" + "github.com/coinbase/rosetta-cli/pkg/results" "github.com/coinbase/rosetta-cli/pkg/tester" "github.com/coinbase/rosetta-sdk-go/fetcher" @@ -83,7 +84,7 @@ func runCheckDataCmd(cmd *cobra.Command, args []string) { _, _, fetchErr := fetcher.InitializeAsserter(ctx, Config.Network) if fetchErr != nil { - tester.ExitData( + results.ExitData( Config, nil, nil, @@ -96,7 +97,7 @@ func runCheckDataCmd(cmd *cobra.Command, args []string) { networkStatus, err := utils.CheckNetworkSupported(ctx, Config.Network, fetcher) if err != nil { - tester.ExitData( + results.ExitData( Config, nil, nil, diff --git a/pkg/tester/construction_results.go b/pkg/results/construction_results.go similarity index 99% rename from pkg/tester/construction_results.go rename to pkg/results/construction_results.go index 0bbaf8c4..fbf8d6e0 100644 --- a/pkg/tester/construction_results.go +++ b/pkg/results/construction_results.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package tester +package results import ( "context" diff --git a/pkg/tester/data_results.go b/pkg/results/data_results.go similarity index 99% rename from pkg/tester/data_results.go rename to pkg/results/data_results.go index 17c49713..75871961 100644 --- a/pkg/tester/data_results.go +++ b/pkg/results/data_results.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package tester +package results import ( "context" diff --git a/pkg/tester/data_results_test.go b/pkg/results/data_results_test.go similarity index 99% rename from pkg/tester/data_results_test.go rename to pkg/results/data_results_test.go index 11d8f653..8b5f5ce9 100644 --- a/pkg/tester/data_results_test.go +++ b/pkg/results/data_results_test.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package tester +package results import ( "context" diff --git a/pkg/tester/construction.go b/pkg/tester/construction.go index 1e170b62..48c287d7 100644 --- a/pkg/tester/construction.go +++ b/pkg/tester/construction.go @@ -27,6 +27,7 @@ import ( "github.com/coinbase/rosetta-cli/configuration" "github.com/coinbase/rosetta-cli/pkg/logger" "github.com/coinbase/rosetta-cli/pkg/processor" + "github.com/coinbase/rosetta-cli/pkg/results" "github.com/coinbase/rosetta-sdk-go/constructor/coordinator" "github.com/coinbase/rosetta-sdk-go/fetcher" @@ -49,6 +50,8 @@ const ( tipWaitInterval = 10 * time.Second ) +var _ http.Handler = (*ConstructionTester)(nil) + // ConstructionTester coordinates the `check:construction` test. type ConstructionTester struct { network *types.NetworkIdentifier @@ -373,7 +376,7 @@ func (t *ConstructionTester) ServeHTTP(w http.ResponseWriter, r *http.Request) { w.Header().Set("Content-Type", "application/json; charset=UTF-8") w.WriteHeader(http.StatusOK) - status := ComputeCheckConstructionStatus( + status := results.ComputeCheckConstructionStatus( r.Context(), t.config, t.counterStorage, @@ -495,7 +498,7 @@ func (t *ConstructionTester) HandleErr( } if !t.reachedEndConditions { - ExitConstruction(t.config, t.counterStorage, t.jobStorage, err, 1) + results.ExitConstruction(t.config, t.counterStorage, t.jobStorage, err, 1) } // We optimistically run the ReturnFunds function on the coordinator @@ -506,5 +509,5 @@ func (t *ConstructionTester) HandleErr( sigListeners, ) - ExitConstruction(t.config, t.counterStorage, t.jobStorage, nil, 0) + results.ExitConstruction(t.config, t.counterStorage, t.jobStorage, nil, 0) } diff --git a/pkg/tester/data.go b/pkg/tester/data.go index 82c2cda0..691c89a7 100644 --- a/pkg/tester/data.go +++ b/pkg/tester/data.go @@ -28,6 +28,7 @@ import ( "github.com/coinbase/rosetta-cli/configuration" "github.com/coinbase/rosetta-cli/pkg/logger" "github.com/coinbase/rosetta-cli/pkg/processor" + "github.com/coinbase/rosetta-cli/pkg/results" "github.com/coinbase/rosetta-sdk-go/fetcher" "github.com/coinbase/rosetta-sdk-go/reconciler" @@ -64,6 +65,8 @@ const ( EndAtTipCheckInterval = 10 * time.Second ) +var _ http.Handler = (*ConstructionTester)(nil) + // DataTester coordinates the `check:data` test. type DataTester struct { network *types.NetworkIdentifier @@ -401,7 +404,7 @@ func (t *DataTester) ServeHTTP(w http.ResponseWriter, r *http.Request) { w.Header().Set("Content-Type", "application/json; charset=UTF-8") w.WriteHeader(http.StatusOK) - status := ComputeCheckDataStatus( + status := results.ComputeCheckDataStatus( r.Context(), t.counterStorage, t.balanceStorage, @@ -560,7 +563,7 @@ func (t *DataTester) HandleErr(ctx context.Context, err error, sigListeners *[]c } if len(t.endCondition) != 0 { - ExitData( + results.ExitData( t.config, t.counterStorage, t.balanceStorage, @@ -573,19 +576,19 @@ func (t *DataTester) HandleErr(ctx context.Context, err error, sigListeners *[]c fmt.Printf("\n") if t.reconcilerHandler.InactiveFailure == nil { - ExitData(t.config, t.counterStorage, t.balanceStorage, err, 1, "", "") + results.ExitData(t.config, t.counterStorage, t.balanceStorage, err, 1, "", "") } if !t.historicalBalanceEnabled { color.Yellow( "Can't find the block missing operations automatically, please enable historical balance lookup", ) - ExitData(t.config, t.counterStorage, t.balanceStorage, err, 1, "", "") + results.ExitData(t.config, t.counterStorage, t.balanceStorage, err, 1, "", "") } if t.config.Data.InactiveDiscrepencySearchDisabled { color.Yellow("Search for inactive reconciliation discrepency is disabled") - ExitData(t.config, t.counterStorage, t.balanceStorage, err, 1, "", "") + results.ExitData(t.config, t.counterStorage, t.balanceStorage, err, 1, "", "") } t.FindMissingOps(ctx, err, sigListeners) @@ -609,7 +612,7 @@ func (t *DataTester) FindMissingOps( ) if err != nil { color.Yellow("%s: could not find block with missing ops", err.Error()) - ExitData(t.config, t.counterStorage, t.balanceStorage, originalErr, 1, "", "") + results.ExitData(t.config, t.counterStorage, t.balanceStorage, originalErr, 1, "", "") } color.Yellow( @@ -619,7 +622,7 @@ func (t *DataTester) FindMissingOps( badBlock.Hash, ) - ExitData(t.config, t.counterStorage, t.balanceStorage, originalErr, 1, "", "") + results.ExitData(t.config, t.counterStorage, t.balanceStorage, originalErr, 1, "", "") } func (t *DataTester) recursiveOpSearch( diff --git a/pkg/tester/general.go b/pkg/tester/general.go index dbc4405d..776070ce 100644 --- a/pkg/tester/general.go +++ b/pkg/tester/general.go @@ -63,7 +63,7 @@ func StartServer( go func() { log.Printf("%s server running on port %d\n", name, port) - server.ListenAndServe() + _ = server.ListenAndServe() }() go func() { @@ -73,7 +73,7 @@ func StartServer( <-ctx.Done() log.Printf("%s server shutting down", name) - server.Shutdown(ctx) + _ = server.Shutdown(ctx) }() return ctx.Err() From 3ad39a56d82128d8be9aa438fe34980cb0fd0356 Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Tue, 29 Sep 2020 21:39:03 -0700 Subject: [PATCH 055/376] Cleanup logger --- pkg/logger/logger.go | 187 ++++++---------------------- pkg/results/construction_results.go | 40 +++++- pkg/results/data_results.go | 6 +- pkg/results/types.go | 6 + 4 files changed, 84 insertions(+), 155 deletions(-) create mode 100644 pkg/results/types.go diff --git a/pkg/logger/logger.go b/pkg/logger/logger.go index 343b7b60..c58deff8 100644 --- a/pkg/logger/logger.go +++ b/pkg/logger/logger.go @@ -18,14 +18,14 @@ import ( "context" "fmt" "log" - "math/big" "os" "path" + "github.com/coinbase/rosetta-cli/pkg/results" + "github.com/coinbase/rosetta-sdk-go/parser" "github.com/coinbase/rosetta-sdk-go/reconciler" "github.com/coinbase/rosetta-sdk-go/statefulsyncer" - "github.com/coinbase/rosetta-sdk-go/storage" "github.com/coinbase/rosetta-sdk-go/types" "github.com/coinbase/rosetta-sdk-go/utils" "github.com/fatih/color" @@ -34,9 +34,6 @@ import ( var _ statefulsyncer.Logger = (*Logger)(nil) const ( - // TimeElapsedCounter tracks the total time elapsed in seconds. - TimeElapsedCounter = "time_elapsed" - // blockStreamFile contains the stream of processed // blocks and whether they were added or removed. blockStreamFile = "blocks.txt" @@ -72,16 +69,12 @@ type Logger struct { logBalanceChanges bool logReconciliation bool - lastStatsMessage string - - CounterStorage *storage.CounterStorage - BalanceStorage *storage.BalanceStorage + lastStatsMessage string + lastProgressMessage string } // NewLogger constructs a new Logger. func NewLogger( - counterStorage *storage.CounterStorage, - balanceStorage *storage.BalanceStorage, logDir string, logBlocks bool, logTransactions bool, @@ -89,8 +82,6 @@ func NewLogger( logReconciliation bool, ) *Logger { return &Logger{ - CounterStorage: counterStorage, - BalanceStorage: balanceStorage, logDir: logDir, logBlocks: logBlocks, logTransactions: logTransactions, @@ -99,168 +90,66 @@ func NewLogger( } } -// LogDataStats logs all data values in CounterStorage. -func (l *Logger) LogDataStats(ctx context.Context) error { - blocks, err := l.CounterStorage.Get(ctx, storage.BlockCounter) - if err != nil { - return fmt.Errorf("%w cannot get block counter", err) - } - - if blocks.Sign() == 0 { // wait for at least 1 block to be processed - return nil - } - - orphans, err := l.CounterStorage.Get(ctx, storage.OrphanCounter) - if err != nil { - return fmt.Errorf("%w cannot get orphan counter", err) - } - - txs, err := l.CounterStorage.Get(ctx, storage.TransactionCounter) - if err != nil { - return fmt.Errorf("%w cannot get transaction counter", err) - } - - ops, err := l.CounterStorage.Get(ctx, storage.OperationCounter) - if err != nil { - return fmt.Errorf("%w cannot get operations counter", err) - } - - activeReconciliations, err := l.CounterStorage.Get(ctx, storage.ActiveReconciliationCounter) - if err != nil { - return fmt.Errorf("%w cannot get active reconciliations counter", err) - } - - inactiveReconciliations, err := l.CounterStorage.Get(ctx, storage.InactiveReconciliationCounter) - if err != nil { - return fmt.Errorf("%w cannot get inactive reconciliations counter", err) +// LogDataStatus logs results.CheckDataStatus. +func (l *Logger) LogDataStatus(ctx context.Context, status *results.CheckDataStatus) { + if status.Stats.Blocks == 0 { // wait for at least 1 block to be processed + return } statsMessage := fmt.Sprintf( - "[STATS] Blocks: %s (Orphaned: %s) Transactions: %s Operations: %s", - blocks.String(), - orphans.String(), - txs.String(), - ops.String(), + "[STATS] Blocks: %d (Orphaned: %d) Transactions: %d Operations: %d Reconciliations: %d (Inactive: %d, Coverage: %f%%)", + status.Stats.Blocks, + status.Stats.Orphans, + status.Stats.Transactions, + status.Stats.Operations, + status.Stats.ActiveReconciliations+status.Stats.InactiveReconciliations, + status.Stats.InactiveReconciliations, + status.Stats.ReconciliationCoverage, ) - if l.BalanceStorage != nil { - coverage, err := l.BalanceStorage.ReconciliationCoverage(ctx, 0) - if err != nil { - return fmt.Errorf("%w: cannot get reconcile coverage", err) - } - - statsMessage = fmt.Sprintf( - "%s Reconciliations: %s (Inactive: %s, Coverage: %f%%)", - statsMessage, - new(big.Int).Add(activeReconciliations, inactiveReconciliations).String(), - inactiveReconciliations.String(), - coverage*utils.OneHundred, - ) - } - // Don't print out the same stats message twice. if statsMessage == l.lastStatsMessage { - return nil + return } l.lastStatsMessage = statsMessage color.Cyan(statsMessage) - return nil -} - -// LogTipEstimate logs information about the remaining blocks to sync. -func (l *Logger) LogTipEstimate(ctx context.Context, tipIndex int64) error { - blocks, err := l.CounterStorage.Get(ctx, storage.BlockCounter) - if err != nil { - return fmt.Errorf("%w cannot get block counter", err) - } - - if blocks.Sign() == 0 { // wait for at least 1 block to be processed - return nil - } - - orphans, err := l.CounterStorage.Get(ctx, storage.OrphanCounter) - if err != nil { - return fmt.Errorf("%w cannot get orphan counter", err) - } - - adjustedBlocks := blocks.Int64() - orphans.Int64() - if tipIndex-adjustedBlocks <= 0 { // return if no blocks to sync - return nil - } - - elapsedTime, err := l.CounterStorage.Get(ctx, TimeElapsedCounter) - if err != nil { - return fmt.Errorf("%w cannot get elapsed time", err) - } - - if elapsedTime.Sign() == 0 { // wait for at least some elapsed time - return nil - } - - blocksPerSecond := new(big.Float).Quo(new(big.Float).SetInt64(adjustedBlocks), new(big.Float).SetInt(elapsedTime)) - blocksPerSecondFloat, _ := blocksPerSecond.Float64() - blocksSynced := new(big.Float).Quo(new(big.Float).SetInt64(adjustedBlocks), new(big.Float).SetInt64(tipIndex)) - blocksSyncedFloat, _ := blocksSynced.Float64() - - statsMessage := fmt.Sprintf( + progressMessage := fmt.Sprintf( "[PROGRESS] Blocks Synced: %d/%d (Completed: %f%%, Rate: %f/second) Time Remaining: %s", - adjustedBlocks, - tipIndex, - blocksSyncedFloat*utils.OneHundred, - blocksPerSecondFloat, - utils.TimeToTip(blocksPerSecondFloat, adjustedBlocks, tipIndex), + status.Progress.Blocks, + status.Progress.Tip, + status.Progress.Completed, + status.Progress.Rate, + status.Progress.TimeRemaining, ) - color.Cyan(statsMessage) - return nil -} - -// LogConstructionStats logs all construction values in CounterStorage. -func (l *Logger) LogConstructionStats(ctx context.Context, inflightTransactions int) error { - transactionsCreated, err := l.CounterStorage.Get(ctx, storage.TransactionsCreatedCounter) - if err != nil { - return fmt.Errorf("%w cannot get transactions created counter", err) - } - - transactionsConfirmed, err := l.CounterStorage.Get(ctx, storage.TransactionsConfirmedCounter) - if err != nil { - return fmt.Errorf("%w cannot get transactions confirmed counter", err) - } - - staleBroadcasts, err := l.CounterStorage.Get(ctx, storage.StaleBroadcastsCounter) - if err != nil { - return fmt.Errorf("%w cannot get stale broadcasts counter", err) - } - - failedBroadcasts, err := l.CounterStorage.Get(ctx, storage.FailedBroadcastsCounter) - if err != nil { - return fmt.Errorf("%w cannot get failed broadcasts counter", err) + // Don't print out the same progress message twice. + if progressMessage == l.lastProgressMessage { + return } - addressesCreated, err := l.CounterStorage.Get(ctx, storage.AddressesCreatedCounter) - if err != nil { - return fmt.Errorf("%w cannot get addresses created counter", err) - } + l.lastProgressMessage = progressMessage + color.Cyan(progressMessage) +} +// LogConstructionStatus logs results.CheckConstructionStatus. +func (l *Logger) LogConstructionStatus(ctx context.Context, status *results.CheckConstructionStatus) { statsMessage := fmt.Sprintf( "[STATS] Transactions Confirmed: %d (Created: %d, In Progress: %d, Stale: %d, Failed: %d) Addresses Created: %d", - transactionsConfirmed, - transactionsCreated, - inflightTransactions, - staleBroadcasts, - failedBroadcasts, - addressesCreated, + status.Stats.TransactionsConfirmed, + status.Stats.TransactionsCreated, + status.Progress.Broadcasting, + status.Stats.StaleBroadcasts, + status.Stats.FailedBroadcasts, + status.Stats.AddressesCreated, ) if statsMessage == l.lastStatsMessage { - return nil + return } l.lastStatsMessage = statsMessage color.Cyan(statsMessage) - - return nil } // LogMemoryStats logs memory usage information. diff --git a/pkg/results/construction_results.go b/pkg/results/construction_results.go index fbf8d6e0..1e3d6ab0 100644 --- a/pkg/results/construction_results.go +++ b/pkg/results/construction_results.go @@ -225,9 +225,43 @@ func ComputeCheckConstructionStats( } } +// CheckConstructionProgress contains the number of +// currently broadcasting transactions and processing +// jobs. +type CheckConstructionProgress struct { + Broadcasting int `json:"broadcasting"` + Processing int `json:"processing"` +} + +// ComputeCheckConstructionProgress computes +// *CheckConstructionProgress. +func ComputeCheckConstructionProgress( + ctx context.Context, + broadcasts *storage.BroadcastStorage, + jobs *storage.JobStorage, +) *CheckConstructionProgress { + inflight, err := broadcasts.GetAllBroadcasts(ctx) + if err != nil { + log.Printf("%s cannot get all broadcasts\n", err.Error()) + return nil + } + + processing, err := jobs.AllProcessing(ctx) + if err != nil { + log.Printf("%s cannot get all jobs\n", err.Error()) + return nil + } + + return &CheckConstructionProgress{ + Broadcasting: len(inflight), + Processing: len(processing), + } +} + // CheckConstructionStatus contains CheckConstructionStats. type CheckConstructionStatus struct { - Stats *CheckConstructionStats `json:"stats"` + Stats *CheckConstructionStats `json:"stats"` + Progress *CheckConstructionProgress `json:"progress"` } // ComputeCheckConstructionStatus returns a populated @@ -236,10 +270,12 @@ func ComputeCheckConstructionStatus( ctx context.Context, config *configuration.Configuration, counters *storage.CounterStorage, + broadcasts *storage.BroadcastStorage, jobs *storage.JobStorage, ) *CheckConstructionStatus { return &CheckConstructionStatus{ - Stats: ComputeCheckConstructionStats(ctx, config, counters, jobs), + Stats: ComputeCheckConstructionStats(ctx, config, counters, jobs), + Progress: ComputeCheckConstructionProgress(ctx, broadcasts, jobs), } } diff --git a/pkg/results/data_results.go b/pkg/results/data_results.go index 75871961..516bc664 100644 --- a/pkg/results/data_results.go +++ b/pkg/results/data_results.go @@ -24,8 +24,6 @@ import ( "strconv" "github.com/coinbase/rosetta-cli/configuration" - "github.com/coinbase/rosetta-cli/pkg/logger" - "github.com/coinbase/rosetta-cli/pkg/processor" "github.com/coinbase/rosetta-sdk-go/asserter" "github.com/coinbase/rosetta-sdk-go/fetcher" @@ -258,7 +256,7 @@ func ComputeCheckDataProgress( return nil } - elapsedTime, err := counters.Get(ctx, logger.TimeElapsedCounter) + elapsedTime, err := counters.Get(ctx, TimeElapsedCounter) if err != nil { fmt.Printf("%s: cannot get elapsed time", err.Error()) return nil @@ -447,7 +445,7 @@ func ReconciliationTest( reconciliationsPerformed bool, ) *bool { relatedErrors := []error{ - processor.ErrReconciliationFailure, + //processor.ErrReconciliationFailure, } reconciliationPass := true for _, relatedError := range relatedErrors { diff --git a/pkg/results/types.go b/pkg/results/types.go new file mode 100644 index 00000000..436069b8 --- /dev/null +++ b/pkg/results/types.go @@ -0,0 +1,6 @@ +package results + +const ( + // TimeElapsedCounter tracks the total time elapsed in seconds. + TimeElapsedCounter = "time_elapsed" +) From 7fe1dc080731fd60d8f938bc41771c2e3d628731 Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Tue, 29 Sep 2020 21:47:42 -0700 Subject: [PATCH 056/376] Continue cleanup --- cmd/check_data.go | 4 --- pkg/processor/reconciler_handler.go | 7 +++-- pkg/tester/construction.go | 11 ++------ pkg/tester/data.go | 43 ++++------------------------- 4 files changed, 13 insertions(+), 52 deletions(-) diff --git a/cmd/check_data.go b/cmd/check_data.go index 2ba41654..1418697f 100644 --- a/cmd/check_data.go +++ b/cmd/check_data.go @@ -146,10 +146,6 @@ func runCheckDataCmd(cmd *cobra.Command, args []string) { return tester.LogMemoryLoop(ctx) }) - g.Go(func() error { - return dataTester.StartProgressLogger(ctx) - }) - g.Go(func() error { return tester.StartServer( ctx, diff --git a/pkg/processor/reconciler_handler.go b/pkg/processor/reconciler_handler.go index 95f58ab3..b6c14083 100644 --- a/pkg/processor/reconciler_handler.go +++ b/pkg/processor/reconciler_handler.go @@ -35,6 +35,7 @@ var ( // ReconcilerHandler implements the Reconciler.Handler interface. type ReconcilerHandler struct { logger *logger.Logger + counterStorage *storage.CounterStorage balanceStorage *storage.BalanceStorage haltOnReconciliationError bool @@ -47,11 +48,13 @@ type ReconcilerHandler struct { // NewReconcilerHandler creates a new ReconcilerHandler. func NewReconcilerHandler( logger *logger.Logger, + counterStorage *storage.CounterStorage, balanceStorage *storage.BalanceStorage, haltOnReconciliationError bool, ) *ReconcilerHandler { return &ReconcilerHandler{ logger: logger, + counterStorage: counterStorage, balanceStorage: balanceStorage, haltOnReconciliationError: haltOnReconciliationError, } @@ -131,13 +134,13 @@ func (h *ReconcilerHandler) ReconciliationSucceeded( ) error { // Update counters if reconciliationType == reconciler.InactiveReconciliation { - _, _ = h.logger.CounterStorage.Update( + _, _ = h.counterStorage.Update( ctx, storage.InactiveReconciliationCounter, big.NewInt(1), ) } else { - _, _ = h.logger.CounterStorage.Update(ctx, storage.ActiveReconciliationCounter, big.NewInt(1)) + _, _ = h.counterStorage.Update(ctx, storage.ActiveReconciliationCounter, big.NewInt(1)) } if err := h.balanceStorage.Reconciled(ctx, account, currency, block); err != nil { diff --git a/pkg/tester/construction.go b/pkg/tester/construction.go index 48c287d7..ed3aebb3 100644 --- a/pkg/tester/construction.go +++ b/pkg/tester/construction.go @@ -92,8 +92,6 @@ func InitializeConstruction( counterStorage := storage.NewCounterStorage(localStore) logger := logger.NewLogger( - counterStorage, - nil, dataPath, false, false, @@ -277,14 +275,10 @@ func (t *ConstructionTester) StartPeriodicLogger( for { select { case <-ctx.Done(): - // Print stats one last time before exiting - inflight, _ := t.broadcastStorage.GetAllBroadcasts(ctx) - _ = t.logger.LogConstructionStats(ctx, len(inflight)) - return ctx.Err() case <-tc.C: - inflight, _ := t.broadcastStorage.GetAllBroadcasts(ctx) - _ = t.logger.LogConstructionStats(ctx, len(inflight)) + status := results.ComputeCheckConstructionStatus(ctx, t.config, t.counterStorage, t.broadcastStorage, t.jobStorage) + t.logger.LogConstructionStatus(ctx, status) } } } @@ -380,6 +374,7 @@ func (t *ConstructionTester) ServeHTTP(w http.ResponseWriter, r *http.Request) { r.Context(), t.config, t.counterStorage, + t.broadcastStorage, t.jobStorage, ) diff --git a/pkg/tester/data.go b/pkg/tester/data.go index 691c89a7..03569cec 100644 --- a/pkg/tester/data.go +++ b/pkg/tester/data.go @@ -165,14 +165,7 @@ func InitializeData( blockStorage := storage.NewBlockStorage(localStore) balanceStorage := storage.NewBalanceStorage(localStore) - loggerBalanceStorage := balanceStorage - if !shouldReconcile(config) { - loggerBalanceStorage = nil - } - logger := logger.NewLogger( - counterStorage, - loggerBalanceStorage, dataPath, config.Data.LogBlocks, config.Data.LogTransactions, @@ -189,6 +182,7 @@ func InitializeData( reconcilerHandler := processor.NewReconcilerHandler( logger, + counterStorage, balanceStorage, !config.Data.IgnoreReconciliationError, ) @@ -354,27 +348,6 @@ func (t *DataTester) StartPeriodicLogger( tc := time.NewTicker(PeriodicLoggingFrequency) defer tc.Stop() - for { - select { - case <-ctx.Done(): - // Print stats one last time before exiting - _ = t.logger.LogDataStats(ctx) - - return ctx.Err() - case <-tc.C: - _ = t.logger.LogDataStats(ctx) - } - } -} - -// StartProgressLogger priunts out periodic -// estimates of sync duration if we are behind tip. -func (t *DataTester) StartProgressLogger( - ctx context.Context, -) error { - tc := time.NewTicker(PeriodicLoggingFrequency) - defer tc.Stop() - for { select { case <-ctx.Done(): @@ -384,17 +357,12 @@ func (t *DataTester) StartProgressLogger( // we can log metrics about the current check:data run. _, _ = t.counterStorage.Update( ctx, - logger.TimeElapsedCounter, + results.TimeElapsedCounter, big.NewInt(periodicLoggingSeconds), ) - status, fetchErr := t.fetcher.NetworkStatusRetry(ctx, t.network, nil) - if fetchErr != nil { - log.Printf("%v: unable to get network status\n", fetchErr.Err) - continue - } - - _ = t.logger.LogTipEstimate(ctx, status.CurrentBlockIdentifier.Index) + status := results.ComputeCheckDataStatus(ctx, t.counterStorage, t.balanceStorage, t.fetcher, t.config.Network) + t.logger.LogDataStatus(ctx, status) } } } @@ -653,8 +621,6 @@ func (t *DataTester) recursiveOpSearch( balanceStorage := storage.NewBalanceStorage(localStore) logger := logger.NewLogger( - counterStorage, - nil, tmpDir, false, false, @@ -671,6 +637,7 @@ func (t *DataTester) recursiveOpSearch( reconcilerHandler := processor.NewReconcilerHandler( logger, + counterStorage, balanceStorage, true, // halt on reconciliation error ) From b810ba4ee0f6adffd7b7ea9a1c9e4c9dbc23bc08 Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Tue, 29 Sep 2020 21:55:50 -0700 Subject: [PATCH 057/376] Continue cleaning --- pkg/logger/logger.go | 2 +- pkg/processor/reconciler_handler.go | 11 +++-------- pkg/results/data_results.go | 2 +- pkg/results/data_results_test.go | 3 +-- pkg/results/types.go | 25 +++++++++++++++++++++++++ 5 files changed, 31 insertions(+), 12 deletions(-) diff --git a/pkg/logger/logger.go b/pkg/logger/logger.go index c58deff8..cde715e5 100644 --- a/pkg/logger/logger.go +++ b/pkg/logger/logger.go @@ -97,7 +97,7 @@ func (l *Logger) LogDataStatus(ctx context.Context, status *results.CheckDataSta } statsMessage := fmt.Sprintf( - "[STATS] Blocks: %d (Orphaned: %d) Transactions: %d Operations: %d Reconciliations: %d (Inactive: %d, Coverage: %f%%)", + "[STATS] Blocks: %d (Orphaned: %d) Transactions: %d Operations: %d Reconciliations: %d (Inactive: %d, Coverage: %f%%)", // nolint:lll status.Stats.Blocks, status.Stats.Orphans, status.Stats.Transactions, diff --git a/pkg/processor/reconciler_handler.go b/pkg/processor/reconciler_handler.go index b6c14083..686311b8 100644 --- a/pkg/processor/reconciler_handler.go +++ b/pkg/processor/reconciler_handler.go @@ -16,22 +16,17 @@ package processor import ( "context" - "errors" "fmt" "math/big" "github.com/coinbase/rosetta-cli/pkg/logger" + "github.com/coinbase/rosetta-cli/pkg/results" "github.com/coinbase/rosetta-sdk-go/reconciler" "github.com/coinbase/rosetta-sdk-go/storage" "github.com/coinbase/rosetta-sdk-go/types" ) -var ( - // ErrReconciliationFailure is returned if reconciliation fails. - ErrReconciliationFailure = errors.New("reconciliation failure") -) - // ReconcilerHandler implements the Reconciler.Handler interface. type ReconcilerHandler struct { logger *logger.Logger @@ -96,7 +91,7 @@ func (h *ReconcilerHandler) ReconciliationFailed( h.InactiveFailureBlock = block return fmt.Errorf( "%w: inactive reconciliation error for %s at %d (computed: %s%s, live: %s%s)", - ErrReconciliationFailure, + results.ErrReconciliationFailure, account.Address, block.Index, computedBalance, @@ -110,7 +105,7 @@ func (h *ReconcilerHandler) ReconciliationFailed( h.ActiveFailureBlock = block return fmt.Errorf( "%w: active reconciliation error for %s at %d (computed: %s%s, live: %s%s)", - ErrReconciliationFailure, + results.ErrReconciliationFailure, account.Address, block.Index, computedBalance, diff --git a/pkg/results/data_results.go b/pkg/results/data_results.go index 516bc664..f1380165 100644 --- a/pkg/results/data_results.go +++ b/pkg/results/data_results.go @@ -445,7 +445,7 @@ func ReconciliationTest( reconciliationsPerformed bool, ) *bool { relatedErrors := []error{ - //processor.ErrReconciliationFailure, + ErrReconciliationFailure, } reconciliationPass := true for _, relatedError := range relatedErrors { diff --git a/pkg/results/data_results_test.go b/pkg/results/data_results_test.go index 8b5f5ce9..68ff2f6d 100644 --- a/pkg/results/data_results_test.go +++ b/pkg/results/data_results_test.go @@ -23,7 +23,6 @@ import ( "testing" "github.com/coinbase/rosetta-cli/configuration" - "github.com/coinbase/rosetta-cli/pkg/processor" "github.com/coinbase/rosetta-sdk-go/asserter" "github.com/coinbase/rosetta-sdk-go/fetcher" @@ -295,7 +294,7 @@ func TestComputeCheckDataResults(t *testing.T) { }, "default configuration, no storage, reconciliation errors": { cfg: configuration.DefaultConfiguration(), - err: []error{processor.ErrReconciliationFailure}, + err: []error{ErrReconciliationFailure}, result: &CheckDataResults{ Tests: &CheckDataTests{ RequestResponse: true, diff --git a/pkg/results/types.go b/pkg/results/types.go index 436069b8..6057ef67 100644 --- a/pkg/results/types.go +++ b/pkg/results/types.go @@ -1,6 +1,31 @@ +// Copyright 2020 Coinbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package results +import ( + "errors" +) + const ( // TimeElapsedCounter tracks the total time elapsed in seconds. TimeElapsedCounter = "time_elapsed" ) + +var ( + // ErrReconciliationFailure is returned if reconciliation fails. + // TODO: Move to reconciler package (had to remove from processor + // to prevent circular dependency) + ErrReconciliationFailure = errors.New("reconciliation failure") +) From ad9d0b376bfa23902a8252639d99f5bf2cc246e9 Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Tue, 29 Sep 2020 21:59:41 -0700 Subject: [PATCH 058/376] Update examples --- examples/configuration/default.json | 1 + 1 file changed, 1 insertion(+) diff --git a/examples/configuration/default.json b/examples/configuration/default.json index 48b5f73b..46869fe8 100644 --- a/examples/configuration/default.json +++ b/examples/configuration/default.json @@ -29,6 +29,7 @@ "inactive_discrepency_search_disabled": false, "balance_tracking_disabled": false, "coin_tracking_disabled": false, + "status_port": 9090, "results_output_file": "", "pruning_disabled": false } From 7b448d5e5901abae080114062bf8bc6cf0bdd95f Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Tue, 29 Sep 2020 22:20:16 -0700 Subject: [PATCH 059/376] Add test for JSON fetch client --- Makefile | 2 +- pkg/results/construction_results.go | 10 ++++++ pkg/results/data_results.go | 10 ++++++ pkg/results/utils.go | 33 +++++++++++++++++++ pkg/results/utils_test.go | 51 +++++++++++++++++++++++++++++ 5 files changed, 105 insertions(+), 1 deletion(-) create mode 100644 pkg/results/utils.go create mode 100644 pkg/results/utils_test.go diff --git a/Makefile b/Makefile index 9a5416cc..6adfc678 100644 --- a/Makefile +++ b/Makefile @@ -12,7 +12,7 @@ ADDLICENCE_SCRIPT=${ADDLICENSE_CMD} -c "Coinbase, Inc." -l "apache" -v GOLINES_CMD=go run github.com/segmentio/golines GOVERALLS_CMD=go run github.com/mattn/goveralls COVERAGE_TEST_DIRECTORIES=./configuration/... ./pkg/constructor/... \ - ./pkg/logger/... ./pkg/scenario/... + ./pkg/logger/... ./pkg/scenario/... ./pkg/results/... TEST_SCRIPT=go test -v ./pkg/... ./configuration/... COVERAGE_TEST_SCRIPT=go test -v ${COVERAGE_TEST_DIRECTORIES} diff --git a/pkg/results/construction_results.go b/pkg/results/construction_results.go index 1e3d6ab0..50139b68 100644 --- a/pkg/results/construction_results.go +++ b/pkg/results/construction_results.go @@ -279,6 +279,16 @@ func ComputeCheckConstructionStatus( } } +// FetchCheckConstructionStatus fetches *CheckConstructionStatus. +func FetchCheckConstructionStatus(url string) (*CheckConstructionStatus, error) { + var status CheckConstructionStatus + if err := JSONFetch(url, &status); err != nil { + return nil, fmt.Errorf("%w: unable to fetch construction status", err) + } + + return &status, nil +} + // ExitConstruction exits check:data, logs the test results to the console, // and to a provided output path. func ExitConstruction( diff --git a/pkg/results/data_results.go b/pkg/results/data_results.go index f1380165..2422946b 100644 --- a/pkg/results/data_results.go +++ b/pkg/results/data_results.go @@ -311,6 +311,16 @@ func ComputeCheckDataStatus( } } +// FetchCheckDataStatus fetches *CheckDataStatus. +func FetchCheckDataStatus(url string) (*CheckDataStatus, error) { + var status CheckDataStatus + if err := JSONFetch(url, &status); err != nil { + return nil, fmt.Errorf("%w: unable to fetch construction status", err) + } + + return &status, nil +} + // CheckDataTests indicates which tests passed. // If a test is nil, it did not apply to the run. // diff --git a/pkg/results/utils.go b/pkg/results/utils.go new file mode 100644 index 00000000..0d7c5645 --- /dev/null +++ b/pkg/results/utils.go @@ -0,0 +1,33 @@ +package results + +import ( + "encoding/json" + "fmt" + "io/ioutil" + "net/http" +) + +// JSONFetch makes a GET request to the URL and marshals +// the response into output. +func JSONFetch(url string, output interface{}) error { + resp, err := http.Get(url) + if err != nil { + return fmt.Errorf("%w: unable to fetch GET %s", err, url) + } + defer resp.Body.Close() + + body, err := ioutil.ReadAll(resp.Body) + if err != nil { + return fmt.Errorf("%w: unable to read body", err) + } + + if resp.StatusCode != http.StatusOK { + return fmt.Errorf("received %d status with body %s", resp.StatusCode, body) + } + + if err := json.Unmarshal(body, output); err != nil { + return fmt.Errorf("%w: unable to unmarshal JSON", err) + } + + return nil +} diff --git a/pkg/results/utils_test.go b/pkg/results/utils_test.go new file mode 100644 index 00000000..9791d579 --- /dev/null +++ b/pkg/results/utils_test.go @@ -0,0 +1,51 @@ +package results + +import ( + "fmt" + "net/http" + "net/http/httptest" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestJSONFetch(t *testing.T) { + var tests = map[string]struct { + status int + body string + + expectedResult map[string]interface{} + expectedError error + }{ + "simple 200": { + status: http.StatusOK, + body: `{"test":"123"}`, + expectedResult: map[string]interface{}{ + "test": "123", + }, + }, + } + + for name, test := range tests { + t.Run(name, func(t *testing.T) { + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + assert.Equal(t, "GET", r.Method) + + w.Header().Set("Content-Type", "application/json; charset=UTF-8") + w.WriteHeader(http.StatusOK) + fmt.Fprintln(w, test.body) + })) + defer ts.Close() + + var obj map[string]interface{} + err := JSONFetch(ts.URL, &obj) + if test.expectedError != nil { + assert.EqualError(t, test.expectedError, err.Error()) + assert.Len(t, obj, 0) + } else { + assert.NoError(t, err) + assert.Equal(t, test.expectedResult, obj) + } + }) + } +} From 9a0ba88b973d31b033e6c0afbda933f236ebca09 Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Tue, 29 Sep 2020 22:25:49 -0700 Subject: [PATCH 060/376] Finish tests for utils --- pkg/results/utils_test.go | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/pkg/results/utils_test.go b/pkg/results/utils_test.go index 9791d579..96d46634 100644 --- a/pkg/results/utils_test.go +++ b/pkg/results/utils_test.go @@ -15,7 +15,7 @@ func TestJSONFetch(t *testing.T) { body string expectedResult map[string]interface{} - expectedError error + expectedError string }{ "simple 200": { status: http.StatusOK, @@ -24,6 +24,16 @@ func TestJSONFetch(t *testing.T) { "test": "123", }, }, + "not 200": { + status: http.StatusUnsupportedMediaType, + body: `hello`, + expectedError: "received 415 status with body hello\n", + }, + "not JSON": { + status: http.StatusOK, + body: `hello`, + expectedError: "invalid character 'h' looking for beginning of value: unable to unmarshal JSON", + }, } for name, test := range tests { @@ -32,15 +42,15 @@ func TestJSONFetch(t *testing.T) { assert.Equal(t, "GET", r.Method) w.Header().Set("Content-Type", "application/json; charset=UTF-8") - w.WriteHeader(http.StatusOK) + w.WriteHeader(test.status) fmt.Fprintln(w, test.body) })) defer ts.Close() var obj map[string]interface{} err := JSONFetch(ts.URL, &obj) - if test.expectedError != nil { - assert.EqualError(t, test.expectedError, err.Error()) + if len(test.expectedError) > 0 { + assert.EqualError(t, err, test.expectedError) assert.Len(t, obj, 0) } else { assert.NoError(t, err) From 090f9c1aae6aa62603d5abb71f86e3ecb6063bdc Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Tue, 29 Sep 2020 22:37:07 -0700 Subject: [PATCH 061/376] nits --- configuration/configuration_test.go | 3 +++ pkg/results/construction_results.go | 4 ++-- pkg/results/data_results.go | 4 ++-- pkg/results/utils.go | 20 +++++++++++++++++--- pkg/results/utils_test.go | 14 ++++++++++++++ 5 files changed, 38 insertions(+), 7 deletions(-) diff --git a/configuration/configuration_test.go b/configuration/configuration_test.go index 30ba5818..b3d58352 100644 --- a/configuration/configuration_test.go +++ b/configuration/configuration_test.go @@ -60,6 +60,7 @@ var ( StaleDepth: 12, BroadcastLimit: 200, BlockBroadcastLimit: 992, + StatusPort: 21, Workflows: append( fakeWorkflows, &job.Workflow{ @@ -75,6 +76,7 @@ var ( ReconciliationDisabled: false, HistoricalBalanceEnabled: &historicalEnabled, StartIndex: &startIndex, + StatusPort: 123, EndConditions: &DataEndConditions{ ReconciliationCoverage: &goodCoverage, }, @@ -153,6 +155,7 @@ func TestLoadConfiguration(t *testing.T) { StaleDepth: DefaultStaleDepth, BroadcastLimit: DefaultBroadcastLimit, BlockBroadcastLimit: DefaultBlockBroadcastLimit, + StatusPort: DefaultStatusPort, Workflows: fakeWorkflows, } diff --git a/pkg/results/construction_results.go b/pkg/results/construction_results.go index 50139b68..9cd9a07d 100644 --- a/pkg/results/construction_results.go +++ b/pkg/results/construction_results.go @@ -280,9 +280,9 @@ func ComputeCheckConstructionStatus( } // FetchCheckConstructionStatus fetches *CheckConstructionStatus. -func FetchCheckConstructionStatus(url string) (*CheckConstructionStatus, error) { +func FetchCheckConstructionStatus(URL string) (*CheckConstructionStatus, error) { var status CheckConstructionStatus - if err := JSONFetch(url, &status); err != nil { + if err := JSONFetch(URL, &status); err != nil { return nil, fmt.Errorf("%w: unable to fetch construction status", err) } diff --git a/pkg/results/data_results.go b/pkg/results/data_results.go index 2422946b..b2db460d 100644 --- a/pkg/results/data_results.go +++ b/pkg/results/data_results.go @@ -312,9 +312,9 @@ func ComputeCheckDataStatus( } // FetchCheckDataStatus fetches *CheckDataStatus. -func FetchCheckDataStatus(url string) (*CheckDataStatus, error) { +func FetchCheckDataStatus(URL string) (*CheckDataStatus, error) { var status CheckDataStatus - if err := JSONFetch(url, &status); err != nil { + if err := JSONFetch(URL, &status); err != nil { return nil, fmt.Errorf("%w: unable to fetch construction status", err) } diff --git a/pkg/results/utils.go b/pkg/results/utils.go index 0d7c5645..9e6d3278 100644 --- a/pkg/results/utils.go +++ b/pkg/results/utils.go @@ -1,3 +1,17 @@ +// Copyright 2020 Coinbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package results import ( @@ -9,10 +23,10 @@ import ( // JSONFetch makes a GET request to the URL and marshals // the response into output. -func JSONFetch(url string, output interface{}) error { - resp, err := http.Get(url) +func JSONFetch(URL string, output interface{}) error { + resp, err := http.Get(URL) if err != nil { - return fmt.Errorf("%w: unable to fetch GET %s", err, url) + return fmt.Errorf("%w: unable to fetch GET %s", err, URL) } defer resp.Body.Close() diff --git a/pkg/results/utils_test.go b/pkg/results/utils_test.go index 96d46634..ed63a9cd 100644 --- a/pkg/results/utils_test.go +++ b/pkg/results/utils_test.go @@ -1,3 +1,17 @@ +// Copyright 2020 Coinbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package results import ( From 52a975a2543bd18bd635803aefeada0830df5e63 Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Tue, 29 Sep 2020 22:37:34 -0700 Subject: [PATCH 062/376] Update version --- cmd/root.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/root.go b/cmd/root.go index 33c1fd18..158e0a16 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -132,6 +132,6 @@ var versionCmd = &cobra.Command{ Use: "version", Short: "Print rosetta-cli version", Run: func(cmd *cobra.Command, args []string) { - fmt.Println("v0.5.5") + fmt.Println("v0.5.6") }, } From 4972b5461bf6d90d322a4a811d61ff4e47726136 Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Tue, 29 Sep 2020 22:46:43 -0700 Subject: [PATCH 063/376] nits --- Makefile | 2 +- pkg/results/construction_results.go | 4 ++-- pkg/results/data_results.go | 4 ++-- pkg/results/utils.go | 6 +++--- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/Makefile b/Makefile index 6adfc678..9a5416cc 100644 --- a/Makefile +++ b/Makefile @@ -12,7 +12,7 @@ ADDLICENCE_SCRIPT=${ADDLICENSE_CMD} -c "Coinbase, Inc." -l "apache" -v GOLINES_CMD=go run github.com/segmentio/golines GOVERALLS_CMD=go run github.com/mattn/goveralls COVERAGE_TEST_DIRECTORIES=./configuration/... ./pkg/constructor/... \ - ./pkg/logger/... ./pkg/scenario/... ./pkg/results/... + ./pkg/logger/... ./pkg/scenario/... TEST_SCRIPT=go test -v ./pkg/... ./configuration/... COVERAGE_TEST_SCRIPT=go test -v ${COVERAGE_TEST_DIRECTORIES} diff --git a/pkg/results/construction_results.go b/pkg/results/construction_results.go index 9cd9a07d..50139b68 100644 --- a/pkg/results/construction_results.go +++ b/pkg/results/construction_results.go @@ -280,9 +280,9 @@ func ComputeCheckConstructionStatus( } // FetchCheckConstructionStatus fetches *CheckConstructionStatus. -func FetchCheckConstructionStatus(URL string) (*CheckConstructionStatus, error) { +func FetchCheckConstructionStatus(url string) (*CheckConstructionStatus, error) { var status CheckConstructionStatus - if err := JSONFetch(URL, &status); err != nil { + if err := JSONFetch(url, &status); err != nil { return nil, fmt.Errorf("%w: unable to fetch construction status", err) } diff --git a/pkg/results/data_results.go b/pkg/results/data_results.go index b2db460d..2422946b 100644 --- a/pkg/results/data_results.go +++ b/pkg/results/data_results.go @@ -312,9 +312,9 @@ func ComputeCheckDataStatus( } // FetchCheckDataStatus fetches *CheckDataStatus. -func FetchCheckDataStatus(URL string) (*CheckDataStatus, error) { +func FetchCheckDataStatus(url string) (*CheckDataStatus, error) { var status CheckDataStatus - if err := JSONFetch(URL, &status); err != nil { + if err := JSONFetch(url, &status); err != nil { return nil, fmt.Errorf("%w: unable to fetch construction status", err) } diff --git a/pkg/results/utils.go b/pkg/results/utils.go index 9e6d3278..d9eeef41 100644 --- a/pkg/results/utils.go +++ b/pkg/results/utils.go @@ -23,10 +23,10 @@ import ( // JSONFetch makes a GET request to the URL and marshals // the response into output. -func JSONFetch(URL string, output interface{}) error { - resp, err := http.Get(URL) +func JSONFetch(url string, output interface{}) error { + resp, err := http.Get(url) // #nosec if err != nil { - return fmt.Errorf("%w: unable to fetch GET %s", err, URL) + return fmt.Errorf("%w: unable to fetch GET %s", err, url) } defer resp.Body.Close() From 47364865dbc4a1e6c9b4226bf164d5bd3cb7341e Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Wed, 30 Sep 2020 16:14:31 -0700 Subject: [PATCH 064/376] Optionally, print all requests and responses to the Construction API --- configuration/configuration.go | 4 ++ pkg/processor/coordinator_helper.go | 59 +++++++++++++++++++++++++++++ pkg/tester/construction.go | 1 + 3 files changed, 64 insertions(+) diff --git a/configuration/configuration.go b/configuration/configuration.go index 4fc5981b..0f6e62f5 100644 --- a/configuration/configuration.go +++ b/configuration/configuration.go @@ -144,6 +144,10 @@ type ConstructionConfiguration struct { // ResultsOutputFile is the absolute filepath of where to save // the results of a check:construction run. ResultsOutputFile string `json:"results_output_file,omitempty"` + + // Quiet is a boolean indicating if all request and response + // logging should be silenced. + Quiet bool `json:"quiet,omitempty"` } // DefaultDataConfiguration returns the default *DataConfiguration diff --git a/pkg/processor/coordinator_helper.go b/pkg/processor/coordinator_helper.go index 8d31138f..b7b13452 100644 --- a/pkg/processor/coordinator_helper.go +++ b/pkg/processor/coordinator_helper.go @@ -17,6 +17,7 @@ package processor import ( "context" "fmt" + "log" "math/big" "github.com/coinbase/rosetta-sdk-go/constructor/coordinator" @@ -26,6 +27,22 @@ import ( "github.com/coinbase/rosetta-sdk-go/types" ) +const ( + request = "REQUEST" + response = "RESPONSE" + reqerror = "ERROR" + queue = "QUEUE" + + constructionDerive = "/construction/derive" + constructionPreprocess = "/construction/preprocess" + constructionMetadata = "/construction/metadata" + constructionPayloads = "/construction/payloads" + constructionParse = "/construction/parse" + constructionCombine = "/construction/combine" + constructionHash = "/construction/hash" + constructionSubmit = "/construction/submit" +) + var _ coordinator.Helper = (*CoordinatorHelper)(nil) // CoordinatorHelper implements the Coordinator.Helper @@ -43,6 +60,10 @@ type CoordinatorHelper struct { counterStorage *storage.CounterStorage balanceStorageHelper *BalanceStorageHelper + + // quiet determines if requests/responses logging + // should be silenced. + quiet bool } // NewCoordinatorHelper returns a new *CoordinatorHelper. @@ -57,6 +78,7 @@ func NewCoordinatorHelper( broadcastStorage *storage.BroadcastStorage, balanceStorageHelper *BalanceStorageHelper, counterStorage *storage.CounterStorage, + quiet bool, ) *CoordinatorHelper { return &CoordinatorHelper{ offlineFetcher: offlineFetcher, @@ -69,6 +91,7 @@ func NewCoordinatorHelper( broadcastStorage: broadcastStorage, counterStorage: counterStorage, balanceStorageHelper: balanceStorageHelper, + quiet: quiet, } } @@ -77,6 +100,20 @@ func (c *CoordinatorHelper) DatabaseTransaction(ctx context.Context) storage.Dat return c.database.NewDatabaseTransaction(ctx, true) } +// verboseLog logs a request or response if c.verbose is true. +func (c *CoordinatorHelper) verboseLog(reqres string, endpoint string, structs ...interface{}) { + if c.quiet { + return + } + + l := fmt.Sprintf("%s %s", reqres, endpoint) + for _, s := range structs { + l = fmt.Sprintf("%s %s", l, types.PrintStruct(s)) + } + + log.Println(l) +} + // Derive returns a new address for a provided publicKey. func (c *CoordinatorHelper) Derive( ctx context.Context, @@ -84,6 +121,7 @@ func (c *CoordinatorHelper) Derive( publicKey *types.PublicKey, metadata map[string]interface{}, ) (*types.AccountIdentifier, map[string]interface{}, error) { + c.verboseLog(request, constructionDerive, networkIdentifier, publicKey, metadata) account, metadata, fetchErr := c.offlineFetcher.ConstructionDerive( ctx, networkIdentifier, @@ -91,9 +129,11 @@ func (c *CoordinatorHelper) Derive( metadata, ) if fetchErr != nil { + c.verboseLog(reqerror, constructionDerive, fetchErr) return nil, nil, fetchErr.Err } + c.verboseLog(response, constructionDerive, account, metadata) return account, metadata, nil } @@ -105,6 +145,7 @@ func (c *CoordinatorHelper) Preprocess( intent []*types.Operation, metadata map[string]interface{}, ) (map[string]interface{}, []*types.AccountIdentifier, error) { + c.verboseLog(request, constructionPreprocess, networkIdentifier, intent, metadata) options, requiredPublicKeys, fetchErr := c.offlineFetcher.ConstructionPreprocess( ctx, networkIdentifier, @@ -113,9 +154,11 @@ func (c *CoordinatorHelper) Preprocess( ) if fetchErr != nil { + c.verboseLog(reqerror, constructionPreprocess, fetchErr) return nil, nil, fetchErr.Err } + c.verboseLog(response, constructionPreprocess, options, requiredPublicKeys) return options, requiredPublicKeys, nil } @@ -127,6 +170,7 @@ func (c *CoordinatorHelper) Metadata( metadataRequest map[string]interface{}, publicKeys []*types.PublicKey, ) (map[string]interface{}, []*types.Amount, error) { + c.verboseLog(request, constructionMetadata, networkIdentifier, metadataRequest, publicKeys) metadata, suggestedFee, fetchErr := c.offlineFetcher.ConstructionMetadata( ctx, networkIdentifier, @@ -135,9 +179,11 @@ func (c *CoordinatorHelper) Metadata( ) if fetchErr != nil { + c.verboseLog(reqerror, constructionMetadata, fetchErr) return nil, nil, fetchErr.Err } + c.verboseLog(response, constructionMetadata, metadata, suggestedFee) return metadata, suggestedFee, nil } @@ -150,6 +196,7 @@ func (c *CoordinatorHelper) Payloads( requiredMetadata map[string]interface{}, publicKeys []*types.PublicKey, ) (string, []*types.SigningPayload, error) { + c.verboseLog(request, constructionPayloads, networkIdentifier, intent, publicKeys) res, payloads, fetchErr := c.offlineFetcher.ConstructionPayloads( ctx, networkIdentifier, @@ -159,9 +206,11 @@ func (c *CoordinatorHelper) Payloads( ) if fetchErr != nil { + c.verboseLog(reqerror, constructionPayloads, fetchErr) return "", nil, fetchErr.Err } + c.verboseLog(response, constructionPayloads, res, payloads) return res, payloads, nil } @@ -173,6 +222,7 @@ func (c *CoordinatorHelper) Parse( signed bool, transaction string, ) ([]*types.Operation, []*types.AccountIdentifier, map[string]interface{}, error) { + c.verboseLog(request, constructionParse, networkIdentifier, signed, transaction) ops, signers, metadata, fetchErr := c.offlineFetcher.ConstructionParse( ctx, networkIdentifier, @@ -181,9 +231,11 @@ func (c *CoordinatorHelper) Parse( ) if fetchErr != nil { + c.verboseLog(reqerror, constructionParse, fetchErr) return nil, nil, nil, fetchErr.Err } + c.verboseLog(response, constructionParse, ops, signers, metadata) return ops, signers, metadata, nil } @@ -195,6 +247,7 @@ func (c *CoordinatorHelper) Combine( unsignedTransaction string, signatures []*types.Signature, ) (string, error) { + c.verboseLog(request, constructionCombine, networkIdentifier, unsignedTransaction, signatures) res, fetchErr := c.offlineFetcher.ConstructionCombine( ctx, networkIdentifier, @@ -203,9 +256,11 @@ func (c *CoordinatorHelper) Combine( ) if fetchErr != nil { + c.verboseLog(reqerror, constructionCombine, fetchErr) return "", fetchErr.Err } + c.verboseLog(response, constructionCombine, res) return res, nil } @@ -216,6 +271,7 @@ func (c *CoordinatorHelper) Hash( networkIdentifier *types.NetworkIdentifier, networkTransaction string, ) (*types.TransactionIdentifier, error) { + c.verboseLog(request, constructionHash, networkIdentifier, networkTransaction) res, fetchErr := c.offlineFetcher.ConstructionHash( ctx, networkIdentifier, @@ -223,9 +279,11 @@ func (c *CoordinatorHelper) Hash( ) if fetchErr != nil { + c.verboseLog(reqerror, constructionHash, fetchErr) return nil, fetchErr.Err } + c.verboseLog(response, constructionHash, res) return res, nil } @@ -348,6 +406,7 @@ func (c *CoordinatorHelper) Broadcast( payload string, confirmationDepth int64, ) error { + c.verboseLog(queue, constructionSubmit, network, intent, transactionIdentifier, payload) return c.broadcastStorage.Broadcast( ctx, dbTx, diff --git a/pkg/tester/construction.go b/pkg/tester/construction.go index ed3aebb3..e286075f 100644 --- a/pkg/tester/construction.go +++ b/pkg/tester/construction.go @@ -202,6 +202,7 @@ func InitializeConstruction( broadcastStorage, balanceStorageHelper, counterStorage, + config.Construction.Quiet, ) coordinatorHandler := processor.NewCoordinatorHandler( From dbc3a6479ebe28a2991be96cf77094063a00ac1b Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Wed, 30 Sep 2020 16:42:02 -0700 Subject: [PATCH 065/376] Add args to output --- pkg/processor/coordinator_helper.go | 113 ++++++++++++++++++++++------ 1 file changed, 88 insertions(+), 25 deletions(-) diff --git a/pkg/processor/coordinator_helper.go b/pkg/processor/coordinator_helper.go index b7b13452..a252e280 100644 --- a/pkg/processor/coordinator_helper.go +++ b/pkg/processor/coordinator_helper.go @@ -41,6 +41,16 @@ const ( constructionCombine = "/construction/combine" constructionHash = "/construction/hash" constructionSubmit = "/construction/submit" + + argNetwork = "network_identifier" + argMetadata = "metadata" + argError = "error" + argAccount = "account_identifier" + argIntent = "intent" + argPublicKeys = "public_keys" + argUnsignedTransaction = "unsigned_transaction" + argTransactionIdentifier = "transaction_identifier" + argNetworkTransaction = "network_transaction" ) var _ coordinator.Helper = (*CoordinatorHelper)(nil) @@ -100,15 +110,20 @@ func (c *CoordinatorHelper) DatabaseTransaction(ctx context.Context) storage.Dat return c.database.NewDatabaseTransaction(ctx, true) } +type arg struct { + name string + val interface{} +} + // verboseLog logs a request or response if c.verbose is true. -func (c *CoordinatorHelper) verboseLog(reqres string, endpoint string, structs ...interface{}) { +func (c *CoordinatorHelper) verboseLog(reqres string, endpoint string, args ...arg) { if c.quiet { return } l := fmt.Sprintf("%s %s", reqres, endpoint) - for _, s := range structs { - l = fmt.Sprintf("%s %s", l, types.PrintStruct(s)) + for _, a := range args { + l = fmt.Sprintf("%s %s:%s", l, a.name, types.PrintStruct(a.val)) } log.Println(l) @@ -121,7 +136,11 @@ func (c *CoordinatorHelper) Derive( publicKey *types.PublicKey, metadata map[string]interface{}, ) (*types.AccountIdentifier, map[string]interface{}, error) { - c.verboseLog(request, constructionDerive, networkIdentifier, publicKey, metadata) + c.verboseLog(request, constructionDerive, + arg{argNetwork, networkIdentifier}, + arg{"public_key", publicKey}, + arg{argMetadata, metadata}, + ) account, metadata, fetchErr := c.offlineFetcher.ConstructionDerive( ctx, networkIdentifier, @@ -129,11 +148,14 @@ func (c *CoordinatorHelper) Derive( metadata, ) if fetchErr != nil { - c.verboseLog(reqerror, constructionDerive, fetchErr) + c.verboseLog(reqerror, constructionDerive, arg{argError, fetchErr}) return nil, nil, fetchErr.Err } - c.verboseLog(response, constructionDerive, account, metadata) + c.verboseLog(response, constructionDerive, + arg{argAccount, account}, + arg{argMetadata, metadata}, + ) return account, metadata, nil } @@ -145,7 +167,11 @@ func (c *CoordinatorHelper) Preprocess( intent []*types.Operation, metadata map[string]interface{}, ) (map[string]interface{}, []*types.AccountIdentifier, error) { - c.verboseLog(request, constructionPreprocess, networkIdentifier, intent, metadata) + c.verboseLog(request, constructionPreprocess, + arg{argNetwork, networkIdentifier}, + arg{argIntent, intent}, + arg{argMetadata, metadata}, + ) options, requiredPublicKeys, fetchErr := c.offlineFetcher.ConstructionPreprocess( ctx, networkIdentifier, @@ -154,11 +180,14 @@ func (c *CoordinatorHelper) Preprocess( ) if fetchErr != nil { - c.verboseLog(reqerror, constructionPreprocess, fetchErr) + c.verboseLog(reqerror, constructionPreprocess, arg{argError, fetchErr}) return nil, nil, fetchErr.Err } - c.verboseLog(response, constructionPreprocess, options, requiredPublicKeys) + c.verboseLog(response, constructionPreprocess, + arg{"options", options}, + arg{"required_public_keys", requiredPublicKeys}, + ) return options, requiredPublicKeys, nil } @@ -170,7 +199,11 @@ func (c *CoordinatorHelper) Metadata( metadataRequest map[string]interface{}, publicKeys []*types.PublicKey, ) (map[string]interface{}, []*types.Amount, error) { - c.verboseLog(request, constructionMetadata, networkIdentifier, metadataRequest, publicKeys) + c.verboseLog(request, constructionMetadata, + arg{argNetwork, networkIdentifier}, + arg{argMetadata, metadataRequest}, + arg{argPublicKeys, publicKeys}, + ) metadata, suggestedFee, fetchErr := c.offlineFetcher.ConstructionMetadata( ctx, networkIdentifier, @@ -179,11 +212,14 @@ func (c *CoordinatorHelper) Metadata( ) if fetchErr != nil { - c.verboseLog(reqerror, constructionMetadata, fetchErr) + c.verboseLog(reqerror, constructionMetadata, arg{argError, fetchErr}) return nil, nil, fetchErr.Err } - c.verboseLog(response, constructionMetadata, metadata, suggestedFee) + c.verboseLog(response, constructionMetadata, + arg{argMetadata, metadata}, + arg{"suggested_fee", suggestedFee}, + ) return metadata, suggestedFee, nil } @@ -196,7 +232,11 @@ func (c *CoordinatorHelper) Payloads( requiredMetadata map[string]interface{}, publicKeys []*types.PublicKey, ) (string, []*types.SigningPayload, error) { - c.verboseLog(request, constructionPayloads, networkIdentifier, intent, publicKeys) + c.verboseLog(request, constructionPayloads, + arg{argNetwork, networkIdentifier}, + arg{argIntent, intent}, + arg{argPublicKeys, publicKeys}, + ) res, payloads, fetchErr := c.offlineFetcher.ConstructionPayloads( ctx, networkIdentifier, @@ -206,11 +246,14 @@ func (c *CoordinatorHelper) Payloads( ) if fetchErr != nil { - c.verboseLog(reqerror, constructionPayloads, fetchErr) + c.verboseLog(reqerror, constructionPayloads, arg{argError, fetchErr}) return "", nil, fetchErr.Err } - c.verboseLog(response, constructionPayloads, res, payloads) + c.verboseLog(response, constructionPayloads, + arg{argUnsignedTransaction, res}, + arg{"payloads", payloads}, + ) return res, payloads, nil } @@ -222,7 +265,11 @@ func (c *CoordinatorHelper) Parse( signed bool, transaction string, ) ([]*types.Operation, []*types.AccountIdentifier, map[string]interface{}, error) { - c.verboseLog(request, constructionParse, networkIdentifier, signed, transaction) + c.verboseLog(request, constructionParse, + arg{argNetwork, networkIdentifier}, + arg{"signed", signed}, + arg{"transaction", transaction}, + ) ops, signers, metadata, fetchErr := c.offlineFetcher.ConstructionParse( ctx, networkIdentifier, @@ -231,11 +278,15 @@ func (c *CoordinatorHelper) Parse( ) if fetchErr != nil { - c.verboseLog(reqerror, constructionParse, fetchErr) + c.verboseLog(reqerror, constructionParse, arg{argError, fetchErr}) return nil, nil, nil, fetchErr.Err } - c.verboseLog(response, constructionParse, ops, signers, metadata) + c.verboseLog(response, constructionParse, + arg{"operations", ops}, + arg{"signers", signers}, + arg{argMetadata, metadata}, + ) return ops, signers, metadata, nil } @@ -247,7 +298,11 @@ func (c *CoordinatorHelper) Combine( unsignedTransaction string, signatures []*types.Signature, ) (string, error) { - c.verboseLog(request, constructionCombine, networkIdentifier, unsignedTransaction, signatures) + c.verboseLog(request, constructionCombine, + arg{argNetwork, networkIdentifier}, + arg{argUnsignedTransaction, unsignedTransaction}, + arg{"signatures", signatures}, + ) res, fetchErr := c.offlineFetcher.ConstructionCombine( ctx, networkIdentifier, @@ -256,11 +311,11 @@ func (c *CoordinatorHelper) Combine( ) if fetchErr != nil { - c.verboseLog(reqerror, constructionCombine, fetchErr) + c.verboseLog(reqerror, constructionCombine, arg{argError, fetchErr}) return "", fetchErr.Err } - c.verboseLog(response, constructionCombine, res) + c.verboseLog(response, constructionCombine, arg{argNetworkTransaction, res}) return res, nil } @@ -271,7 +326,10 @@ func (c *CoordinatorHelper) Hash( networkIdentifier *types.NetworkIdentifier, networkTransaction string, ) (*types.TransactionIdentifier, error) { - c.verboseLog(request, constructionHash, networkIdentifier, networkTransaction) + c.verboseLog(request, constructionHash, + arg{argNetwork, networkIdentifier}, + arg{argNetworkTransaction, networkTransaction}, + ) res, fetchErr := c.offlineFetcher.ConstructionHash( ctx, networkIdentifier, @@ -279,11 +337,11 @@ func (c *CoordinatorHelper) Hash( ) if fetchErr != nil { - c.verboseLog(reqerror, constructionHash, fetchErr) + c.verboseLog(reqerror, constructionHash, arg{argError, fetchErr}) return nil, fetchErr.Err } - c.verboseLog(response, constructionHash, res) + c.verboseLog(response, constructionHash, arg{argTransactionIdentifier, res}) return res, nil } @@ -406,7 +464,12 @@ func (c *CoordinatorHelper) Broadcast( payload string, confirmationDepth int64, ) error { - c.verboseLog(queue, constructionSubmit, network, intent, transactionIdentifier, payload) + c.verboseLog(queue, constructionSubmit, + arg{argNetwork, network}, + arg{argIntent, intent}, + arg{argTransactionIdentifier, transactionIdentifier}, + arg{argNetworkTransaction, payload}, + ) return c.broadcastStorage.Broadcast( ctx, dbTx, From fef9e411f4ad5de174f0795e2b0d4290e7066d88 Mon Sep 17 00:00:00 2001 From: Matheus Degiovani Date: Thu, 1 Oct 2020 10:42:34 -0300 Subject: [PATCH 066/376] logger: Fix possible panic after completion After completing check:data, the Progress field for the current status may be nil, so the old code could panic. --- pkg/logger/logger.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/pkg/logger/logger.go b/pkg/logger/logger.go index cde715e5..4389bb20 100644 --- a/pkg/logger/logger.go +++ b/pkg/logger/logger.go @@ -115,6 +115,11 @@ func (l *Logger) LogDataStatus(ctx context.Context, status *results.CheckDataSta l.lastStatsMessage = statsMessage color.Cyan(statsMessage) + // If Progress is nil, it means we're already done. + if status.Progress == nil { + return + } + progressMessage := fmt.Sprintf( "[PROGRESS] Blocks Synced: %d/%d (Completed: %f%%, Rate: %f/second) Time Remaining: %s", status.Progress.Blocks, From 4bdd9f4e9fe166bc3c017e3c2a3dbaa4ea2f3872 Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Thu, 1 Oct 2020 10:31:05 -0700 Subject: [PATCH 067/376] Overhaul check:construction error handling --- cmd/check_construction.go | 33 ++++++++++++++++++----------- main.go | 10 ++++++--- pkg/results/construction_results.go | 5 ++--- pkg/tester/construction.go | 16 ++++++++------ 4 files changed, 39 insertions(+), 25 deletions(-) diff --git a/cmd/check_construction.go b/cmd/check_construction.go index 05bbbddf..15b1d0c1 100644 --- a/cmd/check_construction.go +++ b/cmd/check_construction.go @@ -16,8 +16,8 @@ package cmd import ( "context" + "errors" "fmt" - "log" "time" "github.com/coinbase/rosetta-cli/pkg/results" @@ -49,13 +49,18 @@ Ethereum. Right now, this tool only supports transfer testing (for both account-based and UTXO-based blockchains). However, we plan to add support for testing arbitrary scenarios (i.e. staking, governance).`, - Run: runCheckConstructionCmd, + RunE: runCheckConstructionCmd, } ) -func runCheckConstructionCmd(cmd *cobra.Command, args []string) { +func runCheckConstructionCmd(cmd *cobra.Command, args []string) error { if Config.Construction == nil { - log.Fatal("construction configuration is missing!") + return results.ExitConstruction( + Config, + nil, + nil, + errors.New("construction configuration is missing"), + ) } ensureDataDirectoryExists() @@ -71,23 +76,23 @@ func runCheckConstructionCmd(cmd *cobra.Command, args []string) { _, _, fetchErr := fetcher.InitializeAsserter(ctx, Config.Network) if fetchErr != nil { - results.ExitConstruction( + cancel() + return results.ExitConstruction( Config, nil, nil, fmt.Errorf("%w: unable to initialize asserter", fetchErr.Err), - 1, ) } _, err := utils.CheckNetworkSupported(ctx, Config.Network, fetcher) if err != nil { - results.ExitConstruction( + cancel() + return results.ExitConstruction( Config, nil, nil, fmt.Errorf("%w: unable to confirm network is supported", err), - 1, ) } @@ -100,19 +105,23 @@ func runCheckConstructionCmd(cmd *cobra.Command, args []string) { &SignalReceived, ) if err != nil { - results.ExitConstruction( + return results.ExitConstruction( Config, nil, nil, fmt.Errorf("%w: unable to initialize construction tester", err), - 1, ) } defer constructionTester.CloseDatabase(ctx) if err := constructionTester.PerformBroadcasts(ctx); err != nil { - log.Fatalf("%s: unable to perform broadcasts", err.Error()) + return results.ExitConstruction( + Config, + nil, + nil, + fmt.Errorf("%w: unable to perform broadcasts", err), + ) } g, ctx := errgroup.WithContext(ctx) @@ -149,5 +158,5 @@ func runCheckConstructionCmd(cmd *cobra.Command, args []string) { go handleSignals(&sigListeners) err = g.Wait() - constructionTester.HandleErr(err, &sigListeners) + return constructionTester.HandleErr(err, &sigListeners) } diff --git a/main.go b/main.go index a68c3eea..1d25c276 100644 --- a/main.go +++ b/main.go @@ -15,13 +15,17 @@ package main import ( - "log" + "os" "github.com/coinbase/rosetta-cli/cmd" + + "github.com/fatih/color" ) func main() { - if err := cmd.Execute(); err != nil { - log.Fatal(err) + err := cmd.Execute() + if err != nil { + color.Red("Command Failed: %s", err.Error()) + os.Exit(1) } } diff --git a/pkg/results/construction_results.go b/pkg/results/construction_results.go index 50139b68..04ec06fa 100644 --- a/pkg/results/construction_results.go +++ b/pkg/results/construction_results.go @@ -296,8 +296,7 @@ func ExitConstruction( counterStorage *storage.CounterStorage, jobStorage *storage.JobStorage, err error, - status int, -) { +) error { results := ComputeCheckConstructionResults( config, err, @@ -309,5 +308,5 @@ func ExitConstruction( results.Output(config.Construction.ResultsOutputFile) } - os.Exit(status) + return err } diff --git a/pkg/tester/construction.go b/pkg/tester/construction.go index e286075f..50fff819 100644 --- a/pkg/tester/construction.go +++ b/pkg/tester/construction.go @@ -21,7 +21,6 @@ import ( "fmt" "log" "net/http" - "os" "time" "github.com/coinbase/rosetta-cli/configuration" @@ -486,15 +485,18 @@ func (t *ConstructionTester) returnFunds( func (t *ConstructionTester) HandleErr( err error, sigListeners *[]context.CancelFunc, -) { +) error { if *t.signalReceived { - color.Red("Check halted") - os.Exit(1) - return + return results.ExitConstruction( + t.config, + t.counterStorage, + t.jobStorage, + errors.New("check halted"), + ) } if !t.reachedEndConditions { - results.ExitConstruction(t.config, t.counterStorage, t.jobStorage, err, 1) + return results.ExitConstruction(t.config, t.counterStorage, t.jobStorage, err) } // We optimistically run the ReturnFunds function on the coordinator @@ -505,5 +507,5 @@ func (t *ConstructionTester) HandleErr( sigListeners, ) - results.ExitConstruction(t.config, t.counterStorage, t.jobStorage, nil, 0) + return results.ExitConstruction(t.config, t.counterStorage, t.jobStorage, nil) } From 38b5bf9696389f2786013280685c0e9a9776e624 Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Thu, 1 Oct 2020 10:42:29 -0700 Subject: [PATCH 068/376] Update check data --- cmd/check_data.go | 14 ++++---- pkg/results/data_results.go | 5 ++- pkg/tester/data.go | 66 +++++++++++++++++++++++++++++-------- 3 files changed, 61 insertions(+), 24 deletions(-) diff --git a/cmd/check_data.go b/cmd/check_data.go index 1418697f..51c93faf 100644 --- a/cmd/check_data.go +++ b/cmd/check_data.go @@ -66,11 +66,11 @@ historical balance disabled to true, you must provide an absolute path to a JSON file containing initial balances with the bootstrap balance config. You can look at the examples folder for an example of what one of these files looks like.`, - Run: runCheckDataCmd, + RunE: runCheckDataCmd, } ) -func runCheckDataCmd(cmd *cobra.Command, args []string) { +func runCheckDataCmd(cmd *cobra.Command, args []string) error { ensureDataDirectoryExists() ctx, cancel := context.WithCancel(context.Background()) @@ -84,12 +84,12 @@ func runCheckDataCmd(cmd *cobra.Command, args []string) { _, _, fetchErr := fetcher.InitializeAsserter(ctx, Config.Network) if fetchErr != nil { - results.ExitData( + cancel() + return results.ExitData( Config, nil, nil, fmt.Errorf("%w: unable to initialize asserter", fetchErr.Err), - 1, "", "", ) @@ -97,12 +97,12 @@ func runCheckDataCmd(cmd *cobra.Command, args []string) { networkStatus, err := utils.CheckNetworkSupported(ctx, Config.Network, fetcher) if err != nil { - results.ExitData( + cancel() + return results.ExitData( Config, nil, nil, fmt.Errorf("%w: unable to confirm network", err), - 1, "", "", ) @@ -166,5 +166,5 @@ func runCheckDataCmd(cmd *cobra.Command, args []string) { // HandleErr will exit if we should not attempt // to find missing operations. - dataTester.HandleErr(ctx, err, &sigListeners) + return dataTester.HandleErr(ctx, err, &sigListeners) } diff --git a/pkg/results/data_results.go b/pkg/results/data_results.go index 2422946b..3266a655 100644 --- a/pkg/results/data_results.go +++ b/pkg/results/data_results.go @@ -571,10 +571,9 @@ func ExitData( counterStorage *storage.CounterStorage, balanceStorage *storage.BalanceStorage, err error, - status int, endCondition configuration.CheckDataEndCondition, endConditionDetail string, -) { +) error { results := ComputeCheckDataResults( config, err, @@ -588,5 +587,5 @@ func ExitData( results.Output(config.Data.ResultsOutputFile) } - os.Exit(status) + return err } diff --git a/pkg/tester/data.go b/pkg/tester/data.go index 03569cec..07b4e568 100644 --- a/pkg/tester/data.go +++ b/pkg/tester/data.go @@ -22,7 +22,6 @@ import ( "log" "math/big" "net/http" - "os" "time" "github.com/coinbase/rosetta-cli/configuration" @@ -513,11 +512,16 @@ func (t *DataTester) WatchEndConditions( // HandleErr is called when `check:data` returns an error. // If historical balance lookups are enabled, HandleErr will attempt to // automatically find any missing balance-changing operations. -func (t *DataTester) HandleErr(ctx context.Context, err error, sigListeners *[]context.CancelFunc) { +func (t *DataTester) HandleErr(ctx context.Context, err error, sigListeners *[]context.CancelFunc) error { if *t.signalReceived { - color.Red("Check halted") - os.Exit(1) - return + return results.ExitData( + t.config, + t.counterStorage, + t.balanceStorage, + errors.New("check halted"), + "", + "", + ) } if (err == nil || errors.Is(err, context.Canceled)) && @@ -531,12 +535,11 @@ func (t *DataTester) HandleErr(ctx context.Context, err error, sigListeners *[]c } if len(t.endCondition) != 0 { - results.ExitData( + return results.ExitData( t.config, t.counterStorage, t.balanceStorage, nil, - 0, t.endCondition, t.endConditionDetail, ) @@ -544,22 +547,43 @@ func (t *DataTester) HandleErr(ctx context.Context, err error, sigListeners *[]c fmt.Printf("\n") if t.reconcilerHandler.InactiveFailure == nil { - results.ExitData(t.config, t.counterStorage, t.balanceStorage, err, 1, "", "") + return results.ExitData( + t.config, + t.counterStorage, + t.balanceStorage, + err, + "", + "", + ) } if !t.historicalBalanceEnabled { color.Yellow( "Can't find the block missing operations automatically, please enable historical balance lookup", ) - results.ExitData(t.config, t.counterStorage, t.balanceStorage, err, 1, "", "") + return results.ExitData( + t.config, + t.counterStorage, + t.balanceStorage, + err, + "", + "", + ) } if t.config.Data.InactiveDiscrepencySearchDisabled { color.Yellow("Search for inactive reconciliation discrepency is disabled") - results.ExitData(t.config, t.counterStorage, t.balanceStorage, err, 1, "", "") + return results.ExitData( + t.config, + t.counterStorage, + t.balanceStorage, + err, + "", + "", + ) } - t.FindMissingOps(ctx, err, sigListeners) + return t.FindMissingOps(ctx, err, sigListeners) } // FindMissingOps logs the types.BlockIdentifier of a block @@ -569,7 +593,7 @@ func (t *DataTester) FindMissingOps( ctx context.Context, originalErr error, sigListeners *[]context.CancelFunc, -) { +) error { color.Cyan("Searching for block with missing operations...hold tight") badBlock, err := t.recursiveOpSearch( ctx, @@ -580,7 +604,14 @@ func (t *DataTester) FindMissingOps( ) if err != nil { color.Yellow("%s: could not find block with missing ops", err.Error()) - results.ExitData(t.config, t.counterStorage, t.balanceStorage, originalErr, 1, "", "") + return results.ExitData( + t.config, + t.counterStorage, + t.balanceStorage, + originalErr, + "", + "", + ) } color.Yellow( @@ -590,7 +621,14 @@ func (t *DataTester) FindMissingOps( badBlock.Hash, ) - results.ExitData(t.config, t.counterStorage, t.balanceStorage, originalErr, 1, "", "") + return results.ExitData( + t.config, + t.counterStorage, + t.balanceStorage, + originalErr, + "", + "", + ) } func (t *DataTester) recursiveOpSearch( From 5a389c93914bfc0d64ec4d67ef7b2970dc66c584 Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Thu, 1 Oct 2020 10:54:26 -0700 Subject: [PATCH 069/376] Convert all commands to runE --- cmd/configuration_create.go | 10 ++++++---- cmd/configuration_validate.go | 12 +++++++----- cmd/utils_asserter_configuration.go | 16 ++++++++++------ cmd/utils_train_zstd.go | 13 +++++++++---- cmd/view_account.go | 18 ++++++++++-------- cmd/view_block.go | 16 +++++++++------- cmd/view_networks.go | 16 ++++++++++------ 7 files changed, 61 insertions(+), 40 deletions(-) diff --git a/cmd/configuration_create.go b/cmd/configuration_create.go index 99e56d9f..0bba6bc5 100644 --- a/cmd/configuration_create.go +++ b/cmd/configuration_create.go @@ -15,7 +15,7 @@ package cmd import ( - "log" + "fmt" "github.com/coinbase/rosetta-cli/configuration" @@ -27,13 +27,15 @@ var ( configurationCreateCmd = &cobra.Command{ Use: "configuration:create", Short: "Create a default configuration file at the provided path", - Run: runConfigurationCreateCmd, + RunE: runConfigurationCreateCmd, Args: cobra.ExactArgs(1), } ) -func runConfigurationCreateCmd(cmd *cobra.Command, args []string) { +func runConfigurationCreateCmd(cmd *cobra.Command, args []string) error { if err := utils.SerializeAndWrite(args[0], configuration.DefaultConfiguration()); err != nil { - log.Fatalf("%s: unable to save configuration file to %s", err.Error(), args[0]) + return fmt.Errorf("%w: unable to save configuration file to %s", err, args[0]) } + + return nil } diff --git a/cmd/configuration_validate.go b/cmd/configuration_validate.go index 178d2799..932e56c3 100644 --- a/cmd/configuration_validate.go +++ b/cmd/configuration_validate.go @@ -15,10 +15,11 @@ package cmd import ( - "log" + "fmt" "github.com/coinbase/rosetta-cli/configuration" + "github.com/fatih/color" "github.com/spf13/cobra" ) @@ -26,16 +27,17 @@ var ( configurationValidateCmd = &cobra.Command{ Use: "configuration:validate", Short: "Ensure a configuration file at the provided path is formatted correctly", - Run: runConfigurationValidateCmd, + RunE: runConfigurationValidateCmd, Args: cobra.ExactArgs(1), } ) -func runConfigurationValidateCmd(cmd *cobra.Command, args []string) { +func runConfigurationValidateCmd(cmd *cobra.Command, args []string) error { _, err := configuration.LoadConfiguration(args[0]) if err != nil { - log.Fatalf("%s: unable to save configuration file to %s", err.Error(), args[0]) + return fmt.Errorf("%w: unable to save configuration file to %s", err, args[0]) } - log.Println("Configuration file validated!") + color.Green("Configuration file validated!") + return nil } diff --git a/cmd/utils_asserter_configuration.go b/cmd/utils_asserter_configuration.go index d92e998f..063a0af5 100644 --- a/cmd/utils_asserter_configuration.go +++ b/cmd/utils_asserter_configuration.go @@ -16,11 +16,12 @@ package cmd import ( "context" - "log" + "fmt" "time" "github.com/coinbase/rosetta-sdk-go/fetcher" "github.com/coinbase/rosetta-sdk-go/utils" + "github.com/fatih/color" "github.com/spf13/cobra" ) @@ -36,12 +37,12 @@ have been added in an update instead of silently erroring. To use this command, simply provide an absolute path as the argument for where the configuration file should be saved (in JSON).`, - Run: runCreateConfigurationCmd, + RunE: runCreateConfigurationCmd, Args: cobra.ExactArgs(1), } ) -func runCreateConfigurationCmd(cmd *cobra.Command, args []string) { +func runCreateConfigurationCmd(cmd *cobra.Command, args []string) error { ctx := context.Background() // Create a new fetcher @@ -55,15 +56,18 @@ func runCreateConfigurationCmd(cmd *cobra.Command, args []string) { // Initialize the fetcher's asserter _, _, fetchErr := newFetcher.InitializeAsserter(ctx, Config.Network) if fetchErr != nil { - log.Fatalf("%s: failed to initialize asserter", fetchErr.Err.Error()) + return fmt.Errorf("%w: failed to initialize asserter", fetchErr.Err) } configuration, err := newFetcher.Asserter.ClientConfiguration() if err != nil { - log.Fatalf("%s: unable to generate spec", err.Error()) + return fmt.Errorf("%w: unable to generate spec", err) } if err := utils.SerializeAndWrite(args[0], configuration); err != nil { - log.Fatalf("%s: unable to serialize asserter configuration", err.Error()) + return fmt.Errorf("%w: unable to serialize asserter configuration", err) } + + color.Green("Configuration file saved!") + return nil } diff --git a/cmd/utils_train_zstd.go b/cmd/utils_train_zstd.go index d1bec9af..a24f926a 100644 --- a/cmd/utils_train_zstd.go +++ b/cmd/utils_train_zstd.go @@ -16,11 +16,13 @@ package cmd import ( "context" + "fmt" "log" "path" "strconv" "github.com/coinbase/rosetta-sdk-go/storage" + "github.com/fatih/color" "github.com/spf13/cobra" ) @@ -43,12 +45,12 @@ The arguments for this command are: You can learn more about dictionary compression on the Zstandard website: https://github.com/facebook/zstd#the-case-for-small-data-compression`, - Run: runTrainZstdCmd, + RunE: runTrainZstdCmd, Args: cobra.MinimumNArgs(trainArgs), } ) -func runTrainZstdCmd(cmd *cobra.Command, args []string) { +func runTrainZstdCmd(cmd *cobra.Command, args []string) error { ctx := context.Background() namespace := args[0] @@ -56,7 +58,7 @@ func runTrainZstdCmd(cmd *cobra.Command, args []string) { dictionaryPath := path.Clean(args[2]) maxItems, err := strconv.Atoi(args[3]) if err != nil { - log.Fatalf("%s: unable to convert max items to integer", err.Error()) + return fmt.Errorf("%w: unable to convert max items to integer", err) } compressorEntries := []*storage.CompressorEntry{} @@ -80,6 +82,9 @@ func runTrainZstdCmd(cmd *cobra.Command, args []string) { compressorEntries, ) if err != nil { - log.Fatal(err) + return fmt.Errorf("%w: badger training failed", err) } + + color.Green("Training successful!") + return nil } diff --git a/cmd/view_account.go b/cmd/view_account.go index 55e1da24..6d844c98 100644 --- a/cmd/view_account.go +++ b/cmd/view_account.go @@ -41,21 +41,21 @@ any account by providing a JSON representation of a types.AccountIdentifier For example, you could run view:account '{"address":"interesting address"}' 1000 to lookup the balance of an interesting address at block 1000. Allowing the address to specified as JSON allows for querying by SubAccountIdentifier.`, - Run: runViewAccountCmd, + RunE: runViewAccountCmd, Args: cobra.MinimumNArgs(1), } ) -func runViewAccountCmd(cmd *cobra.Command, args []string) { +func runViewAccountCmd(cmd *cobra.Command, args []string) error { ctx := context.Background() account := &types.AccountIdentifier{} if err := json.Unmarshal([]byte(args[0]), account); err != nil { - log.Fatal(fmt.Errorf("%w: unable to unmarshal account %s", err, args[0])) + return fmt.Errorf("%w: unable to unmarshal account %s", err, args[0]) } if err := asserter.AccountIdentifier(account); err != nil { - log.Fatal(fmt.Errorf("%w: invalid account identifier %+v", err, account)) + return fmt.Errorf("%w: invalid account identifier %s", err, types.PrintStruct(account)) } // Create a new fetcher @@ -69,19 +69,19 @@ func runViewAccountCmd(cmd *cobra.Command, args []string) { // Initialize the fetcher's asserter _, _, fetchErr := newFetcher.InitializeAsserter(ctx, Config.Network) if fetchErr != nil { - log.Fatal(fetchErr.Err) + return fmt.Errorf("%w: unable to initialize asserter", fetchErr.Err) } _, err := utils.CheckNetworkSupported(ctx, Config.Network, newFetcher) if err != nil { - log.Fatalf("%s: unable to confirm network is supported", err.Error()) + return fmt.Errorf("%w: unable to confirm network is supported", err) } var lookupBlock *types.PartialBlockIdentifier if len(args) > 1 { index, err := strconv.ParseInt(args[1], 10, 64) if err != nil { - log.Fatal(fmt.Errorf("%w: unable to parse index %s", err, args[0])) + return fmt.Errorf("%w: unable to parse index %s", err, args[0]) } lookupBlock = &types.PartialBlockIdentifier{Index: &index} @@ -94,11 +94,13 @@ func runViewAccountCmd(cmd *cobra.Command, args []string) { lookupBlock, ) if fetchErr != nil { - log.Fatal(fmt.Errorf("%w: unable to fetch account %+v", fetchErr.Err, account)) + return fmt.Errorf("%w: unable to fetch account %+v", fetchErr.Err, account) } log.Printf("Amounts: %s\n", types.PrettyPrintStruct(amounts)) log.Printf("Coins: %s\n", types.PrettyPrintStruct(coins)) log.Printf("Metadata: %s\n", types.PrettyPrintStruct(metadata)) log.Printf("Balance Fetched At: %s\n", types.PrettyPrintStruct(block)) + + return nil } diff --git a/cmd/view_block.go b/cmd/view_block.go index 77f72ddd..1a67114c 100644 --- a/cmd/view_block.go +++ b/cmd/view_block.go @@ -41,16 +41,16 @@ of the block is correct before printing. If this command errors, it is likely because the block you are trying to fetch is formatted incorrectly.`, - Run: runViewBlockCmd, + RunE: runViewBlockCmd, Args: cobra.ExactArgs(1), } ) -func runViewBlockCmd(cmd *cobra.Command, args []string) { +func runViewBlockCmd(cmd *cobra.Command, args []string) error { ctx := context.Background() index, err := strconv.ParseInt(args[0], 10, 64) if err != nil { - log.Fatal(fmt.Errorf("%w: unable to parse index %s", err, args[0])) + return fmt.Errorf("%w: unable to parse index %s", err, args[0]) } // Create a new fetcher @@ -68,12 +68,12 @@ func runViewBlockCmd(cmd *cobra.Command, args []string) { // the asserter what are valid responses. _, _, fetchErr := newFetcher.InitializeAsserter(ctx, Config.Network) if fetchErr != nil { - log.Fatal(fetchErr.Err) + return fmt.Errorf("%w: unable to initialize asserter", fetchErr.Err) } _, err = utils.CheckNetworkSupported(ctx, Config.Network, newFetcher) if err != nil { - log.Fatalf("%s: unable to confirm network is supported", err.Error()) + return fmt.Errorf("%w: unable to confirm network is supported", err) } // Fetch the specified block with retries (automatically @@ -93,7 +93,7 @@ func runViewBlockCmd(cmd *cobra.Command, args []string) { }, ) if fetchErr != nil { - log.Fatal(fmt.Errorf("%w: unable to fetch block", fetchErr.Err)) + return fmt.Errorf("%w: unable to fetch block", fetchErr.Err) } log.Printf("Current Block: %s\n", types.PrettyPrintStruct(block)) @@ -103,7 +103,7 @@ func runViewBlockCmd(cmd *cobra.Command, args []string) { p := parser.New(newFetcher.Asserter, func(*types.Operation) bool { return false }) changes, err := p.BalanceChanges(ctx, block, false) if err != nil { - log.Fatal(fmt.Errorf("%w: unable to calculate balance changes", err)) + return fmt.Errorf("%w: unable to calculate balance changes", err) } log.Printf("Balance Changes: %s\n", types.PrettyPrintStruct(changes)) @@ -116,4 +116,6 @@ func runViewBlockCmd(cmd *cobra.Command, args []string) { types.PrettyPrintStruct(parser.GroupOperations(tx)), ) } + + return nil } diff --git a/cmd/view_networks.go b/cmd/view_networks.go index c11c4ffb..7764ef88 100644 --- a/cmd/view_networks.go +++ b/cmd/view_networks.go @@ -16,6 +16,8 @@ package cmd import ( "context" + "errors" + "fmt" "log" "time" @@ -35,11 +37,11 @@ status from all available networks and prints it to the terminal. If this command errors, it is likely because the /network/* endpoints are not formatted correctly.`, - Run: runViewNetworksCmd, + RunE: runViewNetworksCmd, } ) -func runViewNetworksCmd(cmd *cobra.Command, args []string) { +func runViewNetworksCmd(cmd *cobra.Command, args []string) error { ctx := context.Background() f := fetcher.New( @@ -52,11 +54,11 @@ func runViewNetworksCmd(cmd *cobra.Command, args []string) { // Attempt to fetch network list networkList, fetchErr := f.NetworkListRetry(ctx, nil) if fetchErr != nil { - log.Fatalf("%s: unable to fetch network list", fetchErr.Err.Error()) + return fmt.Errorf("%w: unable to fetch network list", fetchErr.Err) } if len(networkList.NetworkIdentifiers) == 0 { - log.Fatal("no networks available") + return errors.New("no networks available") } for _, network := range networkList.NetworkIdentifiers { @@ -67,7 +69,7 @@ func runViewNetworksCmd(cmd *cobra.Command, args []string) { nil, ) if fetchErr != nil { - log.Fatalf("%s: unable to get network options", fetchErr.Err.Error()) + return fmt.Errorf("%w: unable to get network options", fetchErr.Err) } log.Printf("Network options: %s\n", types.PrettyPrintStruct(networkOptions)) @@ -78,9 +80,11 @@ func runViewNetworksCmd(cmd *cobra.Command, args []string) { nil, ) if fetchErr != nil { - log.Fatalf("%s: unable to get network status", fetchErr.Err.Error()) + return fmt.Errorf("%w: unable to get network status", fetchErr.Err) } log.Printf("Network status: %s\n", types.PrettyPrintStruct(networkStatus)) } + + return nil } From dbbab14990e78954a97e1ce04bd7d18760846380 Mon Sep 17 00:00:00 2001 From: Matheus Degiovani Date: Thu, 1 Oct 2020 10:34:39 -0300 Subject: [PATCH 070/376] cmd: Add --cpu-profile and --mem-profile options This adds options to generate a pprof cpu and memory profiles for the main binary. These options are added directly in the root command instead of the configuration file to allow selective use of profiling when using project-wide configuration files. --- cmd/root.go | 86 +++++++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 83 insertions(+), 3 deletions(-) diff --git a/cmd/root.go b/cmd/root.go index 158e0a16..ec52bd66 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -20,6 +20,8 @@ import ( "log" "os" "os/signal" + "runtime" + "runtime/pprof" "syscall" "github.com/coinbase/rosetta-cli/configuration" @@ -31,11 +33,14 @@ import ( var ( rootCmd = &cobra.Command{ - Use: "rosetta-cli", - Short: "CLI for the Rosetta API", + Use: "rosetta-cli", + Short: "CLI for the Rosetta API", + PersistentPreRunE: rootPreRun, } configurationFile string + cpuProfile string + memProfile string // Config is the populated *configuration.Configuration from // the configurationFile. If none is provided, this is set @@ -45,18 +50,81 @@ var ( // SignalReceived is set to true when a signal causes us to exit. This makes // determining the error message to show on exit much more easy. SignalReceived = false + + // profileCleanup is called after the root command is executed to + // cleanup a running cpu profile. + profileCleanup func() ) +// rootPreRun is executed before the root command runs and sets up cpu +// profiling. +// +// Bassed on https://golang.org/pkg/runtime/pprof/#hdr-Profiling_a_Go_program +func rootPreRun(*cobra.Command, []string) error { + if cpuProfile == "" { + return nil + } + + f, err := os.Create(cpuProfile) + if err != nil { + return fmt.Errorf("unable to create CPU profile file: %v", err) + } + if err := pprof.StartCPUProfile(f); err != nil { + if err := f.Close(); err != nil { + log.Printf("error while closing cpu profile file: %v\n", err) + } + return err + } + + profileCleanup = func() { + pprof.StopCPUProfile() + if err := f.Close(); err != nil { + log.Printf("error while closing cpu profile file: %v\n", err) + } + } + return nil +} + +// rootPostRun is executed after the root command runs and performs memory +// profiling. +func rootPostRun() { + if profileCleanup != nil { + profileCleanup() + } + + if memProfile == "" { + return + } + + f, err := os.Create(memProfile) + if err != nil { + log.Printf("error while creating mem-profile file: %v", err) + return + } + defer func() { + if err := f.Close(); err != nil { + log.Printf("error while closing mem-profile file: %v", err) + } + }() + + runtime.GC() + if err := pprof.WriteHeapProfile(f); err != nil { + log.Printf("error while writing heap profile: %v", err) + } +} + // Execute handles all invocations of the // rosetta-cli cmd. func Execute() error { + defer rootPostRun() return rootCmd.Execute() } func init() { cobra.OnInitialize(initConfig) - rootCmd.PersistentFlags().StringVar( + rootFlags := rootCmd.PersistentFlags() + rootFlags.StringVar( &configurationFile, "configuration-file", "", @@ -67,6 +135,18 @@ with the defaults), run rosetta-cli configuration:create. Any fields not populated in the configuration file will be populated with default values.`, ) + rootFlags.StringVar( + &cpuProfile, + "cpu-profile", + "", + `Save the pprof cpu profile in the specified file`, + ) + rootFlags.StringVar( + &memProfile, + "mem-profile", + "", + `Save the pprof mem profile in the specified file`, + ) rootCmd.AddCommand(versionCmd) // Configuration Commands From 53c30db4b8a3a4042bdb3ab2711709bd3074bf96 Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Thu, 1 Oct 2020 12:40:27 -0700 Subject: [PATCH 071/376] Prepare for v0.5.7 release --- cmd/root.go | 5 +++-- go.mod | 2 +- go.sum | 2 ++ 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/cmd/root.go b/cmd/root.go index ec52bd66..717a4001 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -67,7 +67,7 @@ func rootPreRun(*cobra.Command, []string) error { f, err := os.Create(cpuProfile) if err != nil { - return fmt.Errorf("unable to create CPU profile file: %v", err) + return fmt.Errorf("%w: unable to create CPU profile file", err) } if err := pprof.StartCPUProfile(f); err != nil { if err := f.Close(); err != nil { @@ -101,6 +101,7 @@ func rootPostRun() { log.Printf("error while creating mem-profile file: %v", err) return } + defer func() { if err := f.Close(); err != nil { log.Printf("error while closing mem-profile file: %v", err) @@ -212,6 +213,6 @@ var versionCmd = &cobra.Command{ Use: "version", Short: "Print rosetta-cli version", Run: func(cmd *cobra.Command, args []string) { - fmt.Println("v0.5.6") + fmt.Println("v0.5.7") }, } diff --git a/go.mod b/go.mod index 823653ef..36a539dd 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/coinbase/rosetta-cli go 1.13 require ( - github.com/coinbase/rosetta-sdk-go v0.4.8 + github.com/coinbase/rosetta-sdk-go v0.4.9 github.com/fatih/color v1.9.0 github.com/jinzhu/copier v0.0.0-20190924061706-b57f9002281a github.com/olekukonko/tablewriter v0.0.2-0.20190409134802-7e037d187b0c diff --git a/go.sum b/go.sum index a2849f95..e7d73bb7 100644 --- a/go.sum +++ b/go.sum @@ -160,6 +160,8 @@ github.com/coinbase/rosetta-sdk-go v0.4.7 h1:5KFc0CgLMkKamX++hYUFvE58a5/tCn0wSqp github.com/coinbase/rosetta-sdk-go v0.4.7/go.mod h1:8d4iN4VSGvLUzl+jRQlvYSLyS9TeY0QZebneWouizqU= github.com/coinbase/rosetta-sdk-go v0.4.8 h1:+E1TM4q1c5/x/jE9FPI1IZIbNvUWy4tRRgDPLnKzUV4= github.com/coinbase/rosetta-sdk-go v0.4.8/go.mod h1:8d4iN4VSGvLUzl+jRQlvYSLyS9TeY0QZebneWouizqU= +github.com/coinbase/rosetta-sdk-go v0.4.9 h1:Agy5+jrREo4Fa/0Jl0KEOvJDUxaImuCMD0qOxGiUwb4= +github.com/coinbase/rosetta-sdk-go v0.4.9/go.mod h1:8d4iN4VSGvLUzl+jRQlvYSLyS9TeY0QZebneWouizqU= github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk= From 3cffa62221db0ad77ec2bbc3740f990f86795454 Mon Sep 17 00:00:00 2001 From: Luke Champine Date: Fri, 2 Oct 2020 17:10:12 -0400 Subject: [PATCH 072/376] Fix printing of reconciliation coverage percentage --- pkg/logger/logger.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/logger/logger.go b/pkg/logger/logger.go index 4389bb20..f55b1188 100644 --- a/pkg/logger/logger.go +++ b/pkg/logger/logger.go @@ -104,7 +104,7 @@ func (l *Logger) LogDataStatus(ctx context.Context, status *results.CheckDataSta status.Stats.Operations, status.Stats.ActiveReconciliations+status.Stats.InactiveReconciliations, status.Stats.InactiveReconciliations, - status.Stats.ReconciliationCoverage, + status.Stats.ReconciliationCoverage * 100, ) // Don't print out the same stats message twice. From 9a6a75370ac4a757105729b304028521bd7e4c54 Mon Sep 17 00:00:00 2001 From: Luke Champine Date: Fri, 2 Oct 2020 17:21:54 -0400 Subject: [PATCH 073/376] Use constant instead of 100 --- pkg/logger/logger.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/logger/logger.go b/pkg/logger/logger.go index f55b1188..c47f8888 100644 --- a/pkg/logger/logger.go +++ b/pkg/logger/logger.go @@ -104,7 +104,7 @@ func (l *Logger) LogDataStatus(ctx context.Context, status *results.CheckDataSta status.Stats.Operations, status.Stats.ActiveReconciliations+status.Stats.InactiveReconciliations, status.Stats.InactiveReconciliations, - status.Stats.ReconciliationCoverage * 100, + status.Stats.ReconciliationCoverage*utils.OneHundred, ) // Don't print out the same stats message twice. From ab909f0ada44e3de114d382e606872b4d82243b0 Mon Sep 17 00:00:00 2001 From: Kanwar Sahdra <24398939+KNWR@users.noreply.github.com> Date: Mon, 5 Oct 2020 16:32:15 -0400 Subject: [PATCH 074/376] Replace broken guide to commits link --- CONTRIBUTING.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 70312d15..217d1ede 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -49,5 +49,5 @@ be locked to prevent further discussion. All support requests must be made via [our support team][3]. [1]: https://github.com/coinbase/rosetta-cli/issues -[2]: https://medium.com/brigade-engineering/the-secrets-to-great-commit-messages-106fc0a92a25 +[2]: https://chris.beams.io/posts/git-commit/#seven-rules [3]: https://support.coinbase.com/customer/en/portal/articles/2288496-how-can-i-contact-coinbase-support- From 6631edf326e53f690a47439746b7729e5fb6caa6 Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Mon, 5 Oct 2020 18:15:31 -0700 Subject: [PATCH 075/376] update rosetta-sdk-go --- go.mod | 2 +- go.sum | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index 36a539dd..caf28b14 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/coinbase/rosetta-cli go 1.13 require ( - github.com/coinbase/rosetta-sdk-go v0.4.9 + github.com/coinbase/rosetta-sdk-go v0.5.0 github.com/fatih/color v1.9.0 github.com/jinzhu/copier v0.0.0-20190924061706-b57f9002281a github.com/olekukonko/tablewriter v0.0.2-0.20190409134802-7e037d187b0c diff --git a/go.sum b/go.sum index e7d73bb7..43f36168 100644 --- a/go.sum +++ b/go.sum @@ -162,6 +162,8 @@ github.com/coinbase/rosetta-sdk-go v0.4.8 h1:+E1TM4q1c5/x/jE9FPI1IZIbNvUWy4tRRgD github.com/coinbase/rosetta-sdk-go v0.4.8/go.mod h1:8d4iN4VSGvLUzl+jRQlvYSLyS9TeY0QZebneWouizqU= github.com/coinbase/rosetta-sdk-go v0.4.9 h1:Agy5+jrREo4Fa/0Jl0KEOvJDUxaImuCMD0qOxGiUwb4= github.com/coinbase/rosetta-sdk-go v0.4.9/go.mod h1:8d4iN4VSGvLUzl+jRQlvYSLyS9TeY0QZebneWouizqU= +github.com/coinbase/rosetta-sdk-go v0.5.0 h1:WdHJJ7RSUJ1sXwxVAoUb/owH3GYVEK2K3GURIZT3NOA= +github.com/coinbase/rosetta-sdk-go v0.5.0/go.mod h1:QVVeKHWFNb0NyzEY06LxXMAylJkYa7n+Hk03pORr0ws= github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk= @@ -452,6 +454,7 @@ github.com/tidwall/pretty v1.0.2 h1:Z7S3cePv9Jwm1KwS0513MRaoUe3S01WPbLNV40pwWZU= github.com/tidwall/pretty v1.0.2/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= github.com/tidwall/sjson v1.1.1 h1:7h1vk049Jnd5EH9NyzNiEuwYW4b5qgreBbqRC19AS3U= github.com/tidwall/sjson v1.1.1/go.mod h1:yvVuSnpEQv5cYIrO+AT6kw4QVfd5SDZoGIS7/5+fZFs= +github.com/tidwall/sjson v1.1.2/go.mod h1:SEzaDwxiPzKzNfUEO4HbYF/m4UCSJDsGgNqsS1LvdoY= github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/tyler-smith/go-bip39 v1.0.1-0.20181017060643-dbb3b84ba2ef/go.mod h1:sJ5fKU0s6JVwZjjcUEX2zFOnvq0ASQ2K9Zr6cf67kNs= github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc= From 4a8f99f60c1370fb0f32b585c050b0ba384ed13a Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Mon, 5 Oct 2020 20:48:37 -0700 Subject: [PATCH 076/376] Cleanup README --- README.md | 12 ++++-------- go.sum | 1 + 2 files changed, 5 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index ab1884b4..d559e382 100644 --- a/README.md +++ b/README.md @@ -124,10 +124,13 @@ If you have suggestions for more actions, please [open an issue in `rosetta-sdk-go`](https://github.com/coinbase/rosetta-sdk-go/issues)! ##### Workflows -To use the automated Construction API tester, you must implement 2 required `Workflows`: +To use the automated Construction API tester (without prefunded accounts), +you must implement 2 required `Workflows`: * `create_account` * `request_funds` +_If you don't implement these 2 `Workflows`, processing could stall._ + Please note that `create_account` can contain a transaction broadcast if on-chain origination is required for new accounts on your blockchain. @@ -169,13 +172,6 @@ Bitcoin [config](examples/configuration/bitcoin.json). *If this field is not populated or set to `false`, the transaction will be constructed, signed, and broadcast.* -##### Future Work -* DSL for writing `Workflows` (if anyone in the community has ideas for -this, we are all ears!) -* `Workflow` testing tool (to mock `Workflow` before running on network) -* Re-usable components (pre-defined logic that can be used in any `Workflow` - -both user-defined and provided by `rosetta-cli` - #### End Conditions When running the `rosetta-cli` in a CI job, it is usually desired to exit when certain conditions are met (or before then with an exit code of 1). We diff --git a/go.sum b/go.sum index 43f36168..d64decfd 100644 --- a/go.sum +++ b/go.sum @@ -454,6 +454,7 @@ github.com/tidwall/pretty v1.0.2 h1:Z7S3cePv9Jwm1KwS0513MRaoUe3S01WPbLNV40pwWZU= github.com/tidwall/pretty v1.0.2/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= github.com/tidwall/sjson v1.1.1 h1:7h1vk049Jnd5EH9NyzNiEuwYW4b5qgreBbqRC19AS3U= github.com/tidwall/sjson v1.1.1/go.mod h1:yvVuSnpEQv5cYIrO+AT6kw4QVfd5SDZoGIS7/5+fZFs= +github.com/tidwall/sjson v1.1.2 h1:NC5okI+tQ8OG/oyzchvwXXxRxCV/FVdhODbPKkQ25jQ= github.com/tidwall/sjson v1.1.2/go.mod h1:SEzaDwxiPzKzNfUEO4HbYF/m4UCSJDsGgNqsS1LvdoY= github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/tyler-smith/go-bip39 v1.0.1-0.20181017060643-dbb3b84ba2ef/go.mod h1:sJ5fKU0s6JVwZjjcUEX2zFOnvq0ASQ2K9Zr6cf67kNs= From b133a88e1c38d827a84795e8dda77119ff35f5aa Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Mon, 5 Oct 2020 21:11:53 -0700 Subject: [PATCH 077/376] Handle reconciliation exempt --- go.mod | 2 +- go.sum | 2 ++ pkg/processor/reconciler_handler.go | 26 ++++++++++++++++++++++++++ pkg/processor/reconciler_helper.go | 3 +++ 4 files changed, 32 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index caf28b14..161b13dc 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/coinbase/rosetta-cli go 1.13 require ( - github.com/coinbase/rosetta-sdk-go v0.5.0 + github.com/coinbase/rosetta-sdk-go v0.5.1-0.20201006040540-21c3125fd72d github.com/fatih/color v1.9.0 github.com/jinzhu/copier v0.0.0-20190924061706-b57f9002281a github.com/olekukonko/tablewriter v0.0.2-0.20190409134802-7e037d187b0c diff --git a/go.sum b/go.sum index d64decfd..7b3b21e1 100644 --- a/go.sum +++ b/go.sum @@ -164,6 +164,8 @@ github.com/coinbase/rosetta-sdk-go v0.4.9 h1:Agy5+jrREo4Fa/0Jl0KEOvJDUxaImuCMD0q github.com/coinbase/rosetta-sdk-go v0.4.9/go.mod h1:8d4iN4VSGvLUzl+jRQlvYSLyS9TeY0QZebneWouizqU= github.com/coinbase/rosetta-sdk-go v0.5.0 h1:WdHJJ7RSUJ1sXwxVAoUb/owH3GYVEK2K3GURIZT3NOA= github.com/coinbase/rosetta-sdk-go v0.5.0/go.mod h1:QVVeKHWFNb0NyzEY06LxXMAylJkYa7n+Hk03pORr0ws= +github.com/coinbase/rosetta-sdk-go v0.5.1-0.20201006040540-21c3125fd72d h1:GOfiiYEBWClqpXwRR2o7tKDrXUN3GKWv8Qvmi+kblmU= +github.com/coinbase/rosetta-sdk-go v0.5.1-0.20201006040540-21c3125fd72d/go.mod h1:QVVeKHWFNb0NyzEY06LxXMAylJkYa7n+Hk03pORr0ws= github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk= diff --git a/pkg/processor/reconciler_handler.go b/pkg/processor/reconciler_handler.go index 686311b8..b1a5e655 100644 --- a/pkg/processor/reconciler_handler.go +++ b/pkg/processor/reconciler_handler.go @@ -27,6 +27,8 @@ import ( "github.com/coinbase/rosetta-sdk-go/types" ) +var _ reconciler.Handler = (*ReconcilerHandler)(nil) + // ReconcilerHandler implements the Reconciler.Handler interface. type ReconcilerHandler struct { logger *logger.Logger @@ -118,6 +120,30 @@ func (h *ReconcilerHandler) ReconciliationFailed( return nil } +// ReconciliationExempt is called each time a reconciliation fails +// but is considered exempt because of provided []*types.BalanceExemption. +func (h *ReconcilerHandler) ReconciliationExempt( + ctx context.Context, + reconciliationType string, + account *types.AccountIdentifier, + currency *types.Currency, + computedBalance string, + nodeBalance string, + block *types.BlockIdentifier, + exemption *types.BalanceExemption, +) error { + _, _ = h.counterStorage.Update(ctx, storage.ExemptReconciliationCounter, big.NewInt(1)) + + // Although the reconciliation was exempt (non-zero difference that was ignored), + // we still mark the account as being reconciled because the balance was in the range + // specified by exemption. + if err := h.balanceStorage.Reconciled(ctx, account, currency, block); err != nil { + return fmt.Errorf("%w: unable to store updated reconciliation", err) + } + + return nil +} + // ReconciliationSucceeded is called each time a reconciliation succeeds. func (h *ReconcilerHandler) ReconciliationSucceeded( ctx context.Context, diff --git a/pkg/processor/reconciler_helper.go b/pkg/processor/reconciler_helper.go index a23e5630..0b5658c3 100644 --- a/pkg/processor/reconciler_helper.go +++ b/pkg/processor/reconciler_helper.go @@ -19,11 +19,14 @@ import ( "errors" "github.com/coinbase/rosetta-sdk-go/fetcher" + "github.com/coinbase/rosetta-sdk-go/reconciler" "github.com/coinbase/rosetta-sdk-go/storage" "github.com/coinbase/rosetta-sdk-go/types" "github.com/coinbase/rosetta-sdk-go/utils" ) +var _ reconciler.Helper = (*ReconcilerHelper)(nil) + // ReconcilerHelper implements the Reconciler.Helper // interface. type ReconcilerHelper struct { From 96c27ed1b568bfc36f055699537aaddf31b825e6 Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Mon, 5 Oct 2020 21:19:17 -0700 Subject: [PATCH 078/376] Track exempt reconciliations in results --- pkg/results/data_results.go | 25 ++++++++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/pkg/results/data_results.go b/pkg/results/data_results.go index 3266a655..a39a5061 100644 --- a/pkg/results/data_results.go +++ b/pkg/results/data_results.go @@ -96,6 +96,7 @@ type CheckDataStats struct { Operations int64 `json:"operations"` ActiveReconciliations int64 `json:"active_reconciliations"` InactiveReconciliations int64 `json:"inactive_reconciliations"` + ExemptReconciliations int64 `json:"exempt_reconciliations"` ReconciliationCoverage float64 `json:"reconciliation_coverage"` } @@ -127,10 +128,17 @@ func (c *CheckDataStats) Print() { table.Append( []string{ "Inactive Reconciliations", - "# of reconciliation performed on randomly selected accounts", + "# of reconciliations performed on randomly selected accounts", strconv.FormatInt(c.InactiveReconciliations, 10), }, ) + table.Append( + []string{ + "Exempt Reconciliations", + "# of reconciliation failures considered exempt", + strconv.FormatInt(c.ExemptReconciliations, 10), + }, + ) table.Append( []string{ "Reconciliation Coverage", @@ -188,6 +196,12 @@ func ComputeCheckDataStats( return nil } + exemptReconciliations, err := counters.Get(ctx, storage.ExemptReconciliationCounter) + if err != nil { + log.Printf("%s: cannot get exempt reconciliations counter", err.Error()) + return nil + } + stats := &CheckDataStats{ Blocks: blocks.Int64(), Orphans: orphans.Int64(), @@ -195,6 +209,7 @@ func ComputeCheckDataStats( Operations: ops.Int64(), ActiveReconciliations: activeReconciliations.Int64(), InactiveReconciliations: inactiveReconciliations.Int64(), + ExemptReconciliations: exemptReconciliations.Int64(), } if balances != nil { @@ -507,6 +522,14 @@ func ComputeCheckDataTests( if err == nil && inactiveReconciliations.Int64() > 0 { reconciliationsPerformed = true } + + exemptReconciliations, err := counterStorage.Get( + ctx, + storage.ExemptReconciliationCounter, + ) + if err == nil && exemptReconciliations.Int64() > 0 { + reconciliationsPerformed = true + } } return &CheckDataTests{ From ba14f905b870a61decd07d6bc3b99e9a104e921e Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Mon, 5 Oct 2020 21:20:11 -0700 Subject: [PATCH 079/376] Log exempt reconciliations --- pkg/logger/logger.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pkg/logger/logger.go b/pkg/logger/logger.go index c47f8888..74f0f9cf 100644 --- a/pkg/logger/logger.go +++ b/pkg/logger/logger.go @@ -97,13 +97,14 @@ func (l *Logger) LogDataStatus(ctx context.Context, status *results.CheckDataSta } statsMessage := fmt.Sprintf( - "[STATS] Blocks: %d (Orphaned: %d) Transactions: %d Operations: %d Reconciliations: %d (Inactive: %d, Coverage: %f%%)", // nolint:lll + "[STATS] Blocks: %d (Orphaned: %d) Transactions: %d Operations: %d Reconciliations: %d (Inactive: %d, Exempt: %d, Coverage: %f%%)", // nolint:lll status.Stats.Blocks, status.Stats.Orphans, status.Stats.Transactions, status.Stats.Operations, status.Stats.ActiveReconciliations+status.Stats.InactiveReconciliations, status.Stats.InactiveReconciliations, + status.Stats.ExemptReconciliations, status.Stats.ReconciliationCoverage*utils.OneHundred, ) From 4c829d498b5fe30574785c758065581f80d15d62 Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Mon, 5 Oct 2020 21:22:36 -0700 Subject: [PATCH 080/376] Load balance exemptions --- pkg/tester/data.go | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/pkg/tester/data.go b/pkg/tester/data.go index 07b4e568..d8065917 100644 --- a/pkg/tester/data.go +++ b/pkg/tester/data.go @@ -192,16 +192,16 @@ func InitializeData( log.Fatalf("%s: unable to get previously seen accounts", err.Error()) } + networkOptions, fetchErr := fetcher.NetworkOptionsRetry(ctx, network, nil) + if err != nil { + log.Fatalf("%s: unable to get network options", fetchErr.Err.Error()) + } + // Determine if we should perform historical balance lookups var historicalBalanceEnabled bool if config.Data.HistoricalBalanceEnabled != nil { historicalBalanceEnabled = *config.Data.HistoricalBalanceEnabled } else { // we must look it up - networkOptions, fetchErr := fetcher.NetworkOptionsRetry(ctx, network, nil) - if err != nil { - log.Fatalf("%s: unable to get network options", fetchErr.Err.Error()) - } - historicalBalanceEnabled = networkOptions.Allow.HistoricalBalanceLookup } @@ -215,6 +215,7 @@ func InitializeData( reconciler.WithSeenAccounts(seenAccounts), reconciler.WithDebugLogging(config.Data.LogReconciliations), reconciler.WithInactiveFrequency(int64(config.Data.InactiveReconciliationFrequency)), + reconciler.WithBalanceExemptions(networkOptions.Allow.BalanceExemptions), ) blockWorkers := []storage.BlockWorker{} From 4ff07b6809f6e66c3ad6118ba943fc32fa071c4b Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Mon, 5 Oct 2020 21:25:49 -0700 Subject: [PATCH 081/376] Update version --- cmd/root.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/root.go b/cmd/root.go index 717a4001..993a9f88 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -213,6 +213,6 @@ var versionCmd = &cobra.Command{ Use: "version", Short: "Print rosetta-cli version", Run: func(cmd *cobra.Command, args []string) { - fmt.Println("v0.5.7") + fmt.Println("v0.5.8") }, } From 3e82f385e172863ba77e56dc0de685ad7d60d5d1 Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Mon, 5 Oct 2020 21:38:24 -0700 Subject: [PATCH 082/376] Pass existing tests --- configuration/configuration.go | 63 +++++++++++++++++------------ configuration/configuration_test.go | 3 +- 2 files changed, 39 insertions(+), 27 deletions(-) diff --git a/configuration/configuration.go b/configuration/configuration.go index 0f6e62f5..35ab342a 100644 --- a/configuration/configuration.go +++ b/configuration/configuration.go @@ -15,12 +15,14 @@ package configuration import ( + "context" "encoding/hex" "errors" "fmt" "log" "github.com/coinbase/rosetta-sdk-go/asserter" + "github.com/coinbase/rosetta-sdk-go/constructor/dsl" "github.com/coinbase/rosetta-sdk-go/constructor/job" "github.com/coinbase/rosetta-sdk-go/storage" "github.com/coinbase/rosetta-sdk-go/types" @@ -125,10 +127,15 @@ type ConstructionConfiguration struct { PrefundedAccounts []*storage.PrefundedAccount `json:"prefunded_accounts,omitempty"` // Workflows are executed by the rosetta-cli to test - // certain construction flows. Make sure to define a - // "request_funds" and "create_account" workflow. + // certain construction flows. Workflows []*job.Workflow `json:"workflows"` + // ConstructorDSLFile is the path of a Rosetta Constructor + // DSL file (*.ros) that describes which Workflows to test. + // + // DSL Spec: https://github.com/coinbase/rosetta-sdk-go/tree/master/constructor/dsl + ConstructorDSLFile string `json:"constructor_dsl_file"` + // EndConditions is a map of workflow:count that // indicates how many of each workflow should be performed // before check:construction should stop. For example, @@ -440,36 +447,40 @@ func populateMissingFields(config *Configuration) *Configuration { return config } -func assertConstructionConfiguration(config *ConstructionConfiguration) error { +func assertConstructionConfiguration(ctx context.Context, config *ConstructionConfiguration) error { if config == nil { return nil } - seenCreateAccount := false - seenRequestFunds := false - for _, workflow := range config.Workflows { - sawReserved := false - if workflow.Name == string(job.CreateAccount) { - sawReserved = true - seenCreateAccount = true - } + if len(config.Workflows) > 0 && len(config.ConstructorDSLFile) > 0 { + return errors.New("cannot populate both workflows and DSL file path") + } - if workflow.Name == string(job.RequestFunds) { - seenRequestFunds = true - sawReserved = true - } + if len(config.Workflows) == 0 && len(config.ConstructorDSLFile) == 0 { + return errors.New("both workflows and DSL file path are empty") + } - if sawReserved && workflow.Concurrency != job.ReservedWorkflowConcurrency { - return errors.New("reserved workflow must have concurrency 1") + // Parse provided Workflows + for _, workflow := range config.Workflows { + if workflow.Name == string(job.CreateAccount) || workflow.Name == string(job.RequestFunds) { + if workflow.Concurrency != job.ReservedWorkflowConcurrency { + return fmt.Errorf( + "reserved workflow %s must have concurrency %d", + workflow.Name, + job.ReservedWorkflowConcurrency, + ) + } } } - if !seenCreateAccount { - return errors.New("missing create_account workflow") - } + // Compile ConstructorDSLFile and save to Workflows + if len(config.ConstructorDSLFile) > 0 { + compiledWorkflows, err := dsl.Parse(ctx, config.ConstructorDSLFile) + if err != nil { + return fmt.Errorf("%s: compilation failed", types.PrintStruct(err)) + } - if !seenRequestFunds { - return errors.New("missing request_funds workflow") + config.Workflows = compiledWorkflows } for _, account := range config.PrefundedAccounts { @@ -546,7 +557,7 @@ func assertDataConfiguration(config *DataConfiguration) error { return nil } -func assertConfiguration(config *Configuration) error { +func assertConfiguration(ctx context.Context, config *Configuration) error { if err := asserter.NetworkIdentifier(config.Network); err != nil { return fmt.Errorf("%w: invalid network identifier", err) } @@ -555,7 +566,7 @@ func assertConfiguration(config *Configuration) error { return fmt.Errorf("%w: invalid data configuration", err) } - if err := assertConstructionConfiguration(config.Construction); err != nil { + if err := assertConstructionConfiguration(ctx, config.Construction); err != nil { return fmt.Errorf("%w: invalid construction configuration", err) } @@ -564,7 +575,7 @@ func assertConfiguration(config *Configuration) error { // LoadConfiguration returns a parsed and asserted Configuration for running // tests. -func LoadConfiguration(filePath string) (*Configuration, error) { +func LoadConfiguration(ctx context.Context, filePath string) (*Configuration, error) { var configRaw Configuration if err := utils.LoadAndParse(filePath, &configRaw); err != nil { return nil, fmt.Errorf("%w: unable to open configuration file", err) @@ -572,7 +583,7 @@ func LoadConfiguration(filePath string) (*Configuration, error) { config := populateMissingFields(&configRaw) - if err := assertConfiguration(config); err != nil { + if err := assertConfiguration(ctx, config); err != nil { return nil, fmt.Errorf("%w: invalid configuration", err) } diff --git a/configuration/configuration_test.go b/configuration/configuration_test.go index b3d58352..997ab791 100644 --- a/configuration/configuration_test.go +++ b/configuration/configuration_test.go @@ -15,6 +15,7 @@ package configuration import ( + "context" "io/ioutil" "os" "testing" @@ -246,7 +247,7 @@ func TestLoadConfiguration(t *testing.T) { assert.NoError(t, err) // Check if expected fields populated - config, err := LoadConfiguration(tmpfile.Name()) + config, err := LoadConfiguration(context.Background(), tmpfile.Name()) if test.err { assert.Error(t, err) assert.Nil(t, config) From 6a4d1c27f65d170982321b6e8ca167d2c3f9b622 Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Mon, 5 Oct 2020 21:46:44 -0700 Subject: [PATCH 083/376] Pass existing tests --- configuration/configuration_test.go | 45 ++++++++++++++++++++++++++++- configuration/testdata/test.ros | 9 ++++++ 2 files changed, 53 insertions(+), 1 deletion(-) create mode 100644 configuration/testdata/test.ros diff --git a/configuration/configuration_test.go b/configuration/configuration_test.go index 997ab791..0141fe65 100644 --- a/configuration/configuration_test.go +++ b/configuration/configuration_test.go @@ -38,10 +38,22 @@ var ( { Name: string(job.CreateAccount), Concurrency: job.ReservedWorkflowConcurrency, + Scenarios: []*job.Scenario{ + { + Name: "blah", + Actions: []*job.Action{}, + }, + }, }, { Name: string(job.RequestFunds), Concurrency: job.ReservedWorkflowConcurrency, + Scenarios: []*job.Scenario{ + { + Name: "blah", + Actions: []*job.Action{}, + }, + }, }, } whackyConfig = &Configuration{ @@ -163,6 +175,29 @@ func TestLoadConfiguration(t *testing.T) { return cfg }(), }, + "overwrite missing with DSL": { + provided: &Configuration{ + Construction: &ConstructionConfiguration{ + ConstructorDSLFile: "testdata/test.ros", + }, + Data: &DataConfiguration{}, + }, + expected: func() *Configuration { + cfg := DefaultConfiguration() + cfg.Construction = &ConstructionConfiguration{ + OfflineURL: DefaultURL, + MaxOfflineConnections: DefaultMaxOfflineConnections, + StaleDepth: DefaultStaleDepth, + BroadcastLimit: DefaultBroadcastLimit, + BlockBroadcastLimit: DefaultBlockBroadcastLimit, + StatusPort: DefaultStatusPort, + Workflows: fakeWorkflows, + ConstructorDSLFile: "testdata/test.ros", + } + + return cfg + }(), + }, "invalid network": { provided: invalidNetwork, err: true, @@ -216,7 +251,7 @@ func TestLoadConfiguration(t *testing.T) { }, err: true, }, - "missing reserved workflows": { + "empty workflows": { provided: &Configuration{ Construction: &ConstructionConfiguration{ Workflows: []*job.Workflow{}, @@ -224,6 +259,14 @@ func TestLoadConfiguration(t *testing.T) { }, err: true, }, + "non-existent dsl file": { + provided: &Configuration{ + Construction: &ConstructionConfiguration{ + ConstructorDSLFile: "test.ros", + }, + }, + err: true, + }, "multiple end conditions": { provided: multipleEndConditions, expected: func() *Configuration { diff --git a/configuration/testdata/test.ros b/configuration/testdata/test.ros new file mode 100644 index 00000000..86b07cc9 --- /dev/null +++ b/configuration/testdata/test.ros @@ -0,0 +1,9 @@ +create_account(1){ + blah{ + } +} + +request_funds(1){ + blah{ + } +} From 60495af9bb7ed175b629db1027915c2523b4f322 Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Mon, 5 Oct 2020 21:49:41 -0700 Subject: [PATCH 084/376] Add test for non-reserved workflows --- configuration/configuration.go | 20 +++++++++--------- configuration/configuration_test.go | 32 +++++++++++++++++++++++++++++ 2 files changed, 42 insertions(+), 10 deletions(-) diff --git a/configuration/configuration.go b/configuration/configuration.go index 35ab342a..5172a64c 100644 --- a/configuration/configuration.go +++ b/configuration/configuration.go @@ -460,6 +460,16 @@ func assertConstructionConfiguration(ctx context.Context, config *ConstructionCo return errors.New("both workflows and DSL file path are empty") } + // Compile ConstructorDSLFile and save to Workflows + if len(config.ConstructorDSLFile) > 0 { + compiledWorkflows, err := dsl.Parse(ctx, config.ConstructorDSLFile) + if err != nil { + return fmt.Errorf("%s: compilation failed", types.PrintStruct(err)) + } + + config.Workflows = compiledWorkflows + } + // Parse provided Workflows for _, workflow := range config.Workflows { if workflow.Name == string(job.CreateAccount) || workflow.Name == string(job.RequestFunds) { @@ -473,16 +483,6 @@ func assertConstructionConfiguration(ctx context.Context, config *ConstructionCo } } - // Compile ConstructorDSLFile and save to Workflows - if len(config.ConstructorDSLFile) > 0 { - compiledWorkflows, err := dsl.Parse(ctx, config.ConstructorDSLFile) - if err != nil { - return fmt.Errorf("%s: compilation failed", types.PrintStruct(err)) - } - - config.Workflows = compiledWorkflows - } - for _, account := range config.PrefundedAccounts { // Checks that privkey is hex encoded _, err := hex.DecodeString(account.PrivateKeyHex) diff --git a/configuration/configuration_test.go b/configuration/configuration_test.go index 0141fe65..c89820b8 100644 --- a/configuration/configuration_test.go +++ b/configuration/configuration_test.go @@ -198,6 +198,38 @@ func TestLoadConfiguration(t *testing.T) { return cfg }(), }, + "transfer workflow": { + provided: &Configuration{ + Construction: &ConstructionConfiguration{ + Workflows: []*job.Workflow{ + { + Name: "transfer", + Concurrency: 10, + }, + }, + }, + Data: &DataConfiguration{}, + }, + expected: func() *Configuration { + cfg := DefaultConfiguration() + cfg.Construction = &ConstructionConfiguration{ + OfflineURL: DefaultURL, + MaxOfflineConnections: DefaultMaxOfflineConnections, + StaleDepth: DefaultStaleDepth, + BroadcastLimit: DefaultBroadcastLimit, + BlockBroadcastLimit: DefaultBlockBroadcastLimit, + StatusPort: DefaultStatusPort, + Workflows: []*job.Workflow{ + { + Name: "transfer", + Concurrency: 10, + }, + }, + } + + return cfg + }(), + }, "invalid network": { provided: invalidNetwork, err: true, From 576c729b6fbe3de0fa7513cea5981fb00da72f79 Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Mon, 5 Oct 2020 22:07:49 -0700 Subject: [PATCH 085/376] Write rosetta config file --- examples/configuration/ethereum.json | 191 +-------------------------- examples/configuration/ethereum.ros | 121 +++++++++++++++++ 2 files changed, 123 insertions(+), 189 deletions(-) create mode 100644 examples/configuration/ethereum.ros diff --git a/examples/configuration/ethereum.json b/examples/configuration/ethereum.json index 0a3b0a21..295ca755 100644 --- a/examples/configuration/ethereum.json +++ b/examples/configuration/ethereum.json @@ -22,194 +22,7 @@ "broadcast_behind_tip": false, "block_broadcast_limit": 0, "rebroadcast_all": false, - "workflows": [ - { - "name": "request_funds", - "concurrency": 1, - "scenarios": [ - { - "name": "find_account", - "actions": [ - { - "input": "{\"symbol\":\"ETH\", \"decimals\":18}", - "type": "set_variable", - "output_path": "currency" - }, - { - "input": "{\"minimum_balance\":{\"value\": \"0\", \"currency\": {{currency}}}, \"create_limit\":1}", - "type": "find_balance", - "output_path": "random_account" - } - ] - }, - { - "name": "request", - "actions": [ - { - "input": "{\"account_identifier\": {{random_account.account_identifier}}, \"minimum_balance\":{\"value\": \"10000000000000000\", \"currency\": {{currency}}}}", - "type": "find_balance", - "output_path": "loaded_account" - } - ] - } - ] - }, - { - "name": "create_account", - "concurrency": 1, - "scenarios": [ - { - "name": "create_account", - "actions": [ - { - "input": "{\"network\":\"Ropsten\", \"blockchain\":\"Ethereum\"}", - "type": "set_variable", - "output_path": "network" - }, - { - "input": "{\"curve_type\": \"secp256k1\"}", - "type": "generate_key", - "output_path": "key" - }, - { - "input": "{\"network_identifier\": {{network}}, \"public_key\": {{key.public_key}}}", - "type": "derive", - "output_path": "account" - }, - { - "input": "{\"account_identifier\": {{account.account_identifier}}, \"keypair\": {{key}}}", - "type": "save_account" - } - ] - } - ] - }, - { - "name": "transfer", - "concurrency": 10, - "scenarios": [ - { - "name": "transfer", - "actions": [ - { - "input": "{\"network\":\"Ropsten\", \"blockchain\":\"Ethereum\"}", - "type": "set_variable", - "output_path": "transfer.network" - }, - { - "input": "{\"symbol\":\"ETH\", \"decimals\":18}", - "type": "set_variable", - "output_path": "currency" - }, - { - "input": "{\"minimum_balance\":{\"value\": \"10000000000000000\", \"currency\": {{currency}}}}", - "type": "find_balance", - "output_path": "sender" - }, - { - "input": "\"42000000000000\"", - "type": "set_variable", - "output_path": "max_fee" - }, - { - "input": "{\"operation\":\"subtraction\", \"left_value\": {{sender.balance.value}}, \"right_value\": {{max_fee}}}", - "type": "math", - "output_path": "available_amount" - }, - { - "input": "{\"minimum\": \"1\", \"maximum\": {{available_amount}}}", - "type": "random_number", - "output_path": "recipient_amount" - }, - { - "input": "{\"recipient_amount\":{{recipient_amount}}}", - "type": "print_message" - }, - { - "input": "{\"operation\":\"subtraction\", \"left_value\": \"0\", \"right_value\":{{recipient_amount}}}", - "type": "math", - "output_path": "sender_amount" - }, - { - "input": "{\"not_account_identifier\":[{{sender.account_identifier}}], \"minimum_balance\":{\"value\": \"0\", \"currency\": {{currency}}}, \"create_limit\": 100, \"create_probability\": 50}", - "type": "find_balance", - "output_path": "recipient" - }, - { - "input": "\"1\"", - "type": "set_variable", - "output_path": "transfer.confirmation_depth" - }, - { - "input": "[{\"operation_identifier\":{\"index\":0},\"type\":\"transfer\",\"account\":{{sender.account_identifier}},\"amount\":{\"value\":{{sender_amount}},\"currency\":{{currency}}}},{\"operation_identifier\":{\"index\":1},\"type\":\"transfer\",\"account\":{{recipient.account_identifier}},\"amount\":{\"value\":{{recipient_amount}},\"currency\":{{currency}}}}]", - "type": "set_variable", - "output_path": "transfer.operations" - } - ] - } - ] - }, - { - "name": "return_funds", - "concurrency": 10, - "scenarios": [ - { - "name": "transfer", - "actions": [ - { - "input": "{\"network\":\"Ropsten\", \"blockchain\":\"Ethereum\"}", - "type": "set_variable", - "output_path": "transfer.network" - }, - { - "input": "{\"symbol\":\"ETH\", \"decimals\":18}", - "type": "set_variable", - "output_path": "currency" - }, - { - "input": "\"42000000000000\"", - "type": "set_variable", - "output_path": "max_fee" - }, - { - "input": "{\"minimum_balance\":{\"value\": {{max_fee}}, \"currency\": {{currency}}}}", - "type": "find_balance", - "output_path": "sender" - }, - { - "input": "{\"operation\":\"subtraction\", \"left_value\": {{sender.balance.value}}, \"right_value\": {{max_fee}}}", - "type": "math", - "output_path": "available_amount" - }, - { - "input": "{\"return_amount\":{{available_amount}}}", - "type": "print_message" - }, - { - "input": "{\"operation\":\"subtraction\", \"left_value\": \"0\", \"right_value\":{{available_amount}}}", - "type": "math", - "output_path": "sender_amount" - }, - { - "input": "{\"address\":\"0xb41B39479a525AB69e38c701A713D98E3074252c\"}", - "type": "set_variable", - "output_path": "faucet" - }, - { - "input": "\"1\"", - "type": "set_variable", - "output_path": "transfer.confirmation_depth" - }, - { - "input": "[{\"operation_identifier\":{\"index\":0},\"type\":\"transfer\",\"account\":{{sender.account_identifier}},\"amount\":{\"value\":{{sender_amount}},\"currency\":{{currency}}}},{\"operation_identifier\":{\"index\":1},\"type\":\"transfer\",\"account\":{{faucet}},\"amount\":{\"value\":{{available_amount}},\"currency\":{{currency}}}}]", - "type": "set_variable", - "output_path": "transfer.operations" - } - ] - } - ] - } - ], + "constructor_dsl_file": "ethereum.ros", "end_conditions": { "create_account": 10, "transfer": 10 @@ -236,4 +49,4 @@ }, "results_output_file": "" } -} \ No newline at end of file +} diff --git a/examples/configuration/ethereum.ros b/examples/configuration/ethereum.ros new file mode 100644 index 00000000..09e2df3e --- /dev/null +++ b/examples/configuration/ethereum.ros @@ -0,0 +1,121 @@ +request_funds(1){ + find_account{ + currency = {"symbol":"ETH", "decimals":18}; + random_account = find_balance({ + "minimum_balance":{ + "value": "0", + "currency": {{currency}} + }, + "create_limit":1 + }); + }, + request{ + loaded_account = find_balance({ + "account_identifier": {{random_account.account_identifier}}, + "minimum_balance":{ + "value": "10000000000000000", + "currency": {{currency}} + } + }); + } +} + +create_account(1){ + network = {"network":"Ropsten", "blockchain":"Ethereum"}; + key = generate_key({"curve_type": "secp256k1"}); + account = derive({ + "network_identifier": {{network}}, + "public_key": {{key.public_key}} + }); + save_account({ + "account_identifier": {{account.account_identifier}}, + "keypair": {{key}} + }); +} + +transfer(10){ + transfer{ + transfer.network = {"network":"Ropsten", "blockchain":"Ethereum"}; + currency = {"symbol":"ETH", "decimals":18}; + sender = find_balance({ + "minimum_balance":{ + "value": "10000000000000000", + "currency": {{currency}} + } + }); + max_fee = "42000000000000"; + available_amount = {{sender.balance.value}} - {{max_fee}}; + recipient_amount = random_number({"minimum": "1", "maximum": {{available_amount}}}); + print_message({"recipient_amount":{{recipient_amount}}}); + sender_amount = 0 - {{recipient_amount}}; + recipient = find_balance({ + "not_account_identifier":[{{sender.account_identifier}}], + "minimum_balance":{ + "value": "0", + "currency": {{currency}} + }, + "create_limit": 100, + "create_probability": 50 + }); + transfer.confirmation_depth = "1"; + transfer.operations = [ + { + "operation_identifier":{"index":0}, + "type":"transfer", + "account":{{sender.account_identifier}}, + "amount":{ + "value":{{sender_amount}}, + "currency":{{currency}} + } + }, + { + "operation_identifier":{"index":1}, + "type":"transfer", + "account":{{recipient.account_identifier}}, + "amount":{ + "value":{{recipient_amount}}, + "currency":{{currency}} + } + } + ]; + } +} + +return_funds(10){ + transfer{ + transfer.network = {"network":"Ropsten", "blockchain":"Ethereum"}; + currency = {"symbol":"ETH", "decimals":18}; + max_fee = "42000000000000"; + sender = find_balance({ + "minimum_balance":{ + "value": {{max_fee}}, + "currency": {{currency}} + } + }); + available_amount = {{sender.balance.value}} - {{max_fee}}; + print_message({"available_amount":{{available_amount}}}); + sender_amount = 0 - {{available_amount}}; + faucet = {"address":"0xb41B39479a525AB69e38c701A713D98E3074252c"}; + transfer.confirmation_depth = "1"; + transfer.operations = [ + { + "operation_identifier":{"index":0}, + "type":"transfer", + "account":{{sender.account_identifier}}, + "amount":{ + "value":{{sender_amount}}, + "currency":{{currency}} + } + }, + { + "operation_identifier":{"index":1}, + "type":"transfer", + "account":{{faucet}}, + "amount":{ + "value":{{available_amount}}, + "currency":{{currency}} + } + } + ]; + } +} From 99c26f12a4ab6a32264b2e0f5e6bb417ebe406f9 Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Mon, 5 Oct 2020 22:21:42 -0700 Subject: [PATCH 086/376] Update ethereum example to use ethereum.ros --- cmd/check_construction.go | 2 +- cmd/check_data.go | 2 +- cmd/configuration_validate.go | 4 ++-- cmd/root.go | 6 +++++- cmd/utils_asserter_configuration.go | 5 +---- cmd/utils_train_zstd.go | 5 +---- cmd/view_account.go | 9 +++------ cmd/view_block.go | 10 ++++------ cmd/view_networks.go | 9 +++------ examples/configuration/ethereum.json | 2 +- examples/configuration/ethereum.ros | 24 +++++++++++++----------- 11 files changed, 35 insertions(+), 43 deletions(-) diff --git a/cmd/check_construction.go b/cmd/check_construction.go index 15b1d0c1..b7183636 100644 --- a/cmd/check_construction.go +++ b/cmd/check_construction.go @@ -64,7 +64,7 @@ func runCheckConstructionCmd(cmd *cobra.Command, args []string) error { } ensureDataDirectoryExists() - ctx, cancel := context.WithCancel(context.Background()) + ctx, cancel := context.WithCancel(Context) fetcher := fetcher.New( Config.OnlineURL, diff --git a/cmd/check_data.go b/cmd/check_data.go index 51c93faf..55762c99 100644 --- a/cmd/check_data.go +++ b/cmd/check_data.go @@ -72,7 +72,7 @@ of what one of these files looks like.`, func runCheckDataCmd(cmd *cobra.Command, args []string) error { ensureDataDirectoryExists() - ctx, cancel := context.WithCancel(context.Background()) + ctx, cancel := context.WithCancel(Context) fetcher := fetcher.New( Config.OnlineURL, diff --git a/cmd/configuration_validate.go b/cmd/configuration_validate.go index 932e56c3..80844519 100644 --- a/cmd/configuration_validate.go +++ b/cmd/configuration_validate.go @@ -33,9 +33,9 @@ var ( ) func runConfigurationValidateCmd(cmd *cobra.Command, args []string) error { - _, err := configuration.LoadConfiguration(args[0]) + _, err := configuration.LoadConfiguration(Context, args[0]) if err != nil { - return fmt.Errorf("%w: unable to save configuration file to %s", err, args[0]) + return fmt.Errorf("%w: configuration validation failed %s", err, args[0]) } color.Green("Configuration file validated!") diff --git a/cmd/root.go b/cmd/root.go index 993a9f88..7e7e4c7c 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -47,6 +47,9 @@ var ( // to the default settings. Config *configuration.Configuration + // Context is the context to use for this invocation of the cli. + Context context.Context + // SignalReceived is set to true when a signal causes us to exit. This makes // determining the error message to show on exit much more easy. SignalReceived = false @@ -169,11 +172,12 @@ default values.`, } func initConfig() { + Context = context.Background() var err error if len(configurationFile) == 0 { Config = configuration.DefaultConfiguration() } else { - Config, err = configuration.LoadConfiguration(configurationFile) + Config, err = configuration.LoadConfiguration(Context, configurationFile) } if err != nil { log.Fatalf("%s: unable to load configuration", err.Error()) diff --git a/cmd/utils_asserter_configuration.go b/cmd/utils_asserter_configuration.go index 063a0af5..6fde4099 100644 --- a/cmd/utils_asserter_configuration.go +++ b/cmd/utils_asserter_configuration.go @@ -15,7 +15,6 @@ package cmd import ( - "context" "fmt" "time" @@ -43,8 +42,6 @@ the configuration file should be saved (in JSON).`, ) func runCreateConfigurationCmd(cmd *cobra.Command, args []string) error { - ctx := context.Background() - // Create a new fetcher newFetcher := fetcher.New( Config.OnlineURL, @@ -54,7 +51,7 @@ func runCreateConfigurationCmd(cmd *cobra.Command, args []string) error { ) // Initialize the fetcher's asserter - _, _, fetchErr := newFetcher.InitializeAsserter(ctx, Config.Network) + _, _, fetchErr := newFetcher.InitializeAsserter(Context, Config.Network) if fetchErr != nil { return fmt.Errorf("%w: failed to initialize asserter", fetchErr.Err) } diff --git a/cmd/utils_train_zstd.go b/cmd/utils_train_zstd.go index a24f926a..bc715bcd 100644 --- a/cmd/utils_train_zstd.go +++ b/cmd/utils_train_zstd.go @@ -15,7 +15,6 @@ package cmd import ( - "context" "fmt" "log" "path" @@ -51,8 +50,6 @@ website: https://github.com/facebook/zstd#the-case-for-small-data-compression`, ) func runTrainZstdCmd(cmd *cobra.Command, args []string) error { - ctx := context.Background() - namespace := args[0] databasePath := path.Clean(args[1]) dictionaryPath := path.Clean(args[2]) @@ -74,7 +71,7 @@ func runTrainZstdCmd(cmd *cobra.Command, args []string) error { log.Printf("Running zstd training (this could take a while)...") _, _, err = storage.BadgerTrain( - ctx, + Context, namespace, databasePath, dictionaryPath, diff --git a/cmd/view_account.go b/cmd/view_account.go index 6d844c98..e4d33ff6 100644 --- a/cmd/view_account.go +++ b/cmd/view_account.go @@ -15,7 +15,6 @@ package cmd import ( - "context" "encoding/json" "fmt" "log" @@ -47,8 +46,6 @@ address to specified as JSON allows for querying by SubAccountIdentifier.`, ) func runViewAccountCmd(cmd *cobra.Command, args []string) error { - ctx := context.Background() - account := &types.AccountIdentifier{} if err := json.Unmarshal([]byte(args[0]), account); err != nil { return fmt.Errorf("%w: unable to unmarshal account %s", err, args[0]) @@ -67,12 +64,12 @@ func runViewAccountCmd(cmd *cobra.Command, args []string) error { ) // Initialize the fetcher's asserter - _, _, fetchErr := newFetcher.InitializeAsserter(ctx, Config.Network) + _, _, fetchErr := newFetcher.InitializeAsserter(Context, Config.Network) if fetchErr != nil { return fmt.Errorf("%w: unable to initialize asserter", fetchErr.Err) } - _, err := utils.CheckNetworkSupported(ctx, Config.Network, newFetcher) + _, err := utils.CheckNetworkSupported(Context, Config.Network, newFetcher) if err != nil { return fmt.Errorf("%w: unable to confirm network is supported", err) } @@ -88,7 +85,7 @@ func runViewAccountCmd(cmd *cobra.Command, args []string) error { } block, amounts, coins, metadata, fetchErr := newFetcher.AccountBalanceRetry( - ctx, + Context, Config.Network, account, lookupBlock, diff --git a/cmd/view_block.go b/cmd/view_block.go index 1a67114c..f6c5de5e 100644 --- a/cmd/view_block.go +++ b/cmd/view_block.go @@ -15,7 +15,6 @@ package cmd import ( - "context" "fmt" "log" "strconv" @@ -47,7 +46,6 @@ fetch is formatted incorrectly.`, ) func runViewBlockCmd(cmd *cobra.Command, args []string) error { - ctx := context.Background() index, err := strconv.ParseInt(args[0], 10, 64) if err != nil { return fmt.Errorf("%w: unable to parse index %s", err, args[0]) @@ -66,12 +64,12 @@ func runViewBlockCmd(cmd *cobra.Command, args []string) error { // Behind the scenes this makes a call to get the // network status and uses the response to inform // the asserter what are valid responses. - _, _, fetchErr := newFetcher.InitializeAsserter(ctx, Config.Network) + _, _, fetchErr := newFetcher.InitializeAsserter(Context, Config.Network) if fetchErr != nil { return fmt.Errorf("%w: unable to initialize asserter", fetchErr.Err) } - _, err = utils.CheckNetworkSupported(ctx, Config.Network, newFetcher) + _, err = utils.CheckNetworkSupported(Context, Config.Network, newFetcher) if err != nil { return fmt.Errorf("%w: unable to confirm network is supported", err) } @@ -86,7 +84,7 @@ func runViewBlockCmd(cmd *cobra.Command, args []string) error { // to fully populate the block by fetching all these // transactions. block, fetchErr := newFetcher.BlockRetry( - ctx, + Context, Config.Network, &types.PartialBlockIdentifier{ Index: &index, @@ -101,7 +99,7 @@ func runViewBlockCmd(cmd *cobra.Command, args []string) error { // Print out all balance changes in a given block. This does NOT exempt // any operations/accounts from parsing. p := parser.New(newFetcher.Asserter, func(*types.Operation) bool { return false }) - changes, err := p.BalanceChanges(ctx, block, false) + changes, err := p.BalanceChanges(Context, block, false) if err != nil { return fmt.Errorf("%w: unable to calculate balance changes", err) } diff --git a/cmd/view_networks.go b/cmd/view_networks.go index 7764ef88..6dab0d82 100644 --- a/cmd/view_networks.go +++ b/cmd/view_networks.go @@ -15,7 +15,6 @@ package cmd import ( - "context" "errors" "fmt" "log" @@ -42,8 +41,6 @@ not formatted correctly.`, ) func runViewNetworksCmd(cmd *cobra.Command, args []string) error { - ctx := context.Background() - f := fetcher.New( Config.OnlineURL, fetcher.WithRetryElapsedTime(time.Duration(Config.RetryElapsedTime)*time.Second), @@ -52,7 +49,7 @@ func runViewNetworksCmd(cmd *cobra.Command, args []string) error { ) // Attempt to fetch network list - networkList, fetchErr := f.NetworkListRetry(ctx, nil) + networkList, fetchErr := f.NetworkListRetry(Context, nil) if fetchErr != nil { return fmt.Errorf("%w: unable to fetch network list", fetchErr.Err) } @@ -64,7 +61,7 @@ func runViewNetworksCmd(cmd *cobra.Command, args []string) error { for _, network := range networkList.NetworkIdentifiers { color.Cyan(types.PrettyPrintStruct(network)) networkOptions, fetchErr := f.NetworkOptions( - ctx, + Context, network, nil, ) @@ -75,7 +72,7 @@ func runViewNetworksCmd(cmd *cobra.Command, args []string) error { log.Printf("Network options: %s\n", types.PrettyPrintStruct(networkOptions)) networkStatus, fetchErr := f.NetworkStatusRetry( - ctx, + Context, network, nil, ) diff --git a/examples/configuration/ethereum.json b/examples/configuration/ethereum.json index 295ca755..7592bc93 100644 --- a/examples/configuration/ethereum.json +++ b/examples/configuration/ethereum.json @@ -22,7 +22,7 @@ "broadcast_behind_tip": false, "block_broadcast_limit": 0, "rebroadcast_all": false, - "constructor_dsl_file": "ethereum.ros", + "constructor_dsl_file": "examples/configuration/ethereum.ros", "end_conditions": { "create_account": 10, "transfer": 10 diff --git a/examples/configuration/ethereum.ros b/examples/configuration/ethereum.ros index 09e2df3e..35a5d0c8 100644 --- a/examples/configuration/ethereum.ros +++ b/examples/configuration/ethereum.ros @@ -21,16 +21,18 @@ request_funds(1){ } create_account(1){ - network = {"network":"Ropsten", "blockchain":"Ethereum"}; - key = generate_key({"curve_type": "secp256k1"}); - account = derive({ - "network_identifier": {{network}}, - "public_key": {{key.public_key}} - }); - save_account({ - "account_identifier": {{account.account_identifier}}, - "keypair": {{key}} - }); + create{ + network = {"network":"Ropsten", "blockchain":"Ethereum"}; + key = generate_key({"curve_type": "secp256k1"}); + account = derive({ + "network_identifier": {{network}}, + "public_key": {{key.public_key}} + }); + save_account({ + "account_identifier": {{account.account_identifier}}, + "keypair": {{key}} + }); + } } transfer(10){ @@ -43,7 +45,7 @@ transfer(10){ "currency": {{currency}} } }); - max_fee = "42000000000000"; + max_fee = set_variable("42000000000000"); available_amount = {{sender.balance.value}} - {{max_fee}}; recipient_amount = random_number({"minimum": "1", "maximum": {{available_amount}}}); print_message({"recipient_amount":{{recipient_amount}}}); From a97ce065c6842fa3914ac0d0c5db8f90fd410264 Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Tue, 6 Oct 2020 09:00:38 -0700 Subject: [PATCH 087/376] Load rosetta file relative location --- configuration/configuration.go | 325 +++------------------------ configuration/configuration_test.go | 30 ++- configuration/types.go | 302 +++++++++++++++++++++++++ examples/configuration/ethereum.json | 2 +- 4 files changed, 352 insertions(+), 307 deletions(-) create mode 100644 configuration/types.go diff --git a/configuration/configuration.go b/configuration/configuration.go index 5172a64c..8fa7c93e 100644 --- a/configuration/configuration.go +++ b/configuration/configuration.go @@ -20,143 +20,16 @@ import ( "errors" "fmt" "log" + "path" "github.com/coinbase/rosetta-sdk-go/asserter" "github.com/coinbase/rosetta-sdk-go/constructor/dsl" "github.com/coinbase/rosetta-sdk-go/constructor/job" - "github.com/coinbase/rosetta-sdk-go/storage" "github.com/coinbase/rosetta-sdk-go/types" "github.com/coinbase/rosetta-sdk-go/utils" "github.com/fatih/color" ) -// CheckDataEndCondition is a type of "successful" end -// for the "check:data" method. -type CheckDataEndCondition string - -const ( - // IndexEndCondition is used to indicate that the index end condition - // has been met. - IndexEndCondition CheckDataEndCondition = "Index End Condition" - - // DurationEndCondition is used to indicate that the duration - // end condition has been met. - DurationEndCondition CheckDataEndCondition = "Duration End Condition" - - // TipEndCondition is used to indicate that the tip end condition - // has been met. - TipEndCondition CheckDataEndCondition = "Tip End Condition" - - // ReconciliationCoverageEndCondition is used to indicate that the reconciliation - // coverage end condition has been met. - ReconciliationCoverageEndCondition CheckDataEndCondition = "Reconciliation Coverage End Condition" -) - -// Default Configuration Values -const ( - DefaultURL = "http://localhost:8080" - DefaultTimeout = 10 - DefaultMaxRetries = 5 - DefaultMaxOnlineConnections = 120 // most OS have a default limit of 128 - DefaultMaxOfflineConnections = 4 // we shouldn't need many connections for construction - DefaultMaxSyncConcurrency = 64 - DefaultActiveReconciliationConcurrency = 16 - DefaultInactiveReconciliationConcurrency = 4 - DefaultInactiveReconciliationFrequency = 250 - DefaultConfirmationDepth = 10 - DefaultStaleDepth = 30 - DefaultBroadcastLimit = 3 - DefaultTipDelay = 300 - DefaultBlockBroadcastLimit = 5 - DefaultStatusPort = 9090 - - // ETH Defaults - EthereumIDBlockchain = "Ethereum" - EthereumIDNetwork = "Ropsten" -) - -// Default Configuration Values -var ( - EthereumNetwork = &types.NetworkIdentifier{ - Blockchain: EthereumIDBlockchain, - Network: EthereumIDNetwork, - } -) - -// ConstructionConfiguration contains all configurations -// to run check:construction. -type ConstructionConfiguration struct { - // OfflineURL is the URL of a Rosetta API implementation in "offline mode". - OfflineURL string `json:"offline_url"` - - // MaxOffineConnections is the maximum number of open connections that the offline - // fetcher will open. - MaxOfflineConnections int `json:"max_offline_connections"` - - // StaleDepth is the number of blocks to wait before attempting - // to rebroadcast after not finding a transaction on-chain. - StaleDepth int64 `json:"stale_depth"` - - // BroadcastLimit is the number of times to attempt re-broadcast - // before giving up on a transaction broadcast. - BroadcastLimit int `json:"broadcast_limit"` - - // IgnoreBroadcastFailures determines if we should exit when there - // are broadcast failures (that surpass the BroadcastLimit). - IgnoreBroadcastFailures bool `json:"ignore_broadcast_failures"` - - // ClearBroadcasts indicates if all pending broadcasts should - // be removed from BroadcastStorage on restart. - ClearBroadcasts bool `json:"clear_broadcasts"` - - // BroadcastBehindTip indicates if we should broadcast transactions - // when we are behind tip (as defined by TipDelay). - BroadcastBehindTip bool `json:"broadcast_behind_tip"` - - // BlockBroadcastLimit is the number of transactions to attempt - // broadcast in a single block. When there are many pending - // broadcasts, it may make sense to limit the number of broadcasts. - BlockBroadcastLimit int `json:"block_broadcast_limit"` - - // RebroadcastAll indicates if all pending broadcasts should be - // rebroadcast from BroadcastStorage on restart. - RebroadcastAll bool `json:"rebroadcast_all"` - - // PrefundedAccounts is an array of prefunded accounts - // to use while testing. - PrefundedAccounts []*storage.PrefundedAccount `json:"prefunded_accounts,omitempty"` - - // Workflows are executed by the rosetta-cli to test - // certain construction flows. - Workflows []*job.Workflow `json:"workflows"` - - // ConstructorDSLFile is the path of a Rosetta Constructor - // DSL file (*.ros) that describes which Workflows to test. - // - // DSL Spec: https://github.com/coinbase/rosetta-sdk-go/tree/master/constructor/dsl - ConstructorDSLFile string `json:"constructor_dsl_file"` - - // EndConditions is a map of workflow:count that - // indicates how many of each workflow should be performed - // before check:construction should stop. For example, - // {"create_account": 5} indicates that 5 "create_account" - // workflows should be performed before stopping. - EndConditions map[string]int `json:"end_conditions,omitempty"` - - // StatusPort allows the caller to query a running check:construction - // test to get stats about progress. This can be used instead - // of parsing logs to populate some sort of status dashboard. - StatusPort uint `json:"status_port,omitempty"` - - // ResultsOutputFile is the absolute filepath of where to save - // the results of a check:construction run. - ResultsOutputFile string `json:"results_output_file,omitempty"` - - // Quiet is a boolean indicating if all request and response - // logging should be silenced. - Quiet bool `json:"quiet,omitempty"` -} - // DefaultDataConfiguration returns the default *DataConfiguration // for running `check:data`. func DefaultDataConfiguration() *DataConfiguration { @@ -184,172 +57,6 @@ func DefaultConfiguration() *Configuration { } } -// DataEndConditions contains all the conditions for the syncer to stop -// when running check:data. -type DataEndConditions struct { - // Index configures the syncer to stop once reaching a particular block height. - Index *int64 `json:"index,omitempty"` - - // Tip configures the syncer to stop once it reached the tip. - // Make sure to configure `tip_delay` if you use this end - // condition. - Tip *bool `json:"tip,omitempty"` - - // Duration configures the syncer to stop after running - // for Duration seconds. - Duration *uint64 `json:"duration,omitempty"` - - // ReconciliationCoverage configures the syncer to stop - // once it has reached tip AND some proportion of - // all addresses have been reconciled at an index >= - // to when tip was first reached. The range of inputs - // for this condition are [0.0, 1.0]. - ReconciliationCoverage *float64 `json:"reconciliation_coverage,omitempty"` -} - -// DataConfiguration contains all configurations to run check:data. -type DataConfiguration struct { - // ActiveReconciliationConcurrency is the concurrency to use while fetching accounts - // during active reconciliation. - ActiveReconciliationConcurrency uint64 `json:"active_reconciliation_concurrency"` - - // InactiveReconciliationConcurrency is the concurrency to use while fetching accounts - // during inactive reconciliation. - InactiveReconciliationConcurrency uint64 `json:"inactive_reconciliation_concurrency"` - - // InactiveReconciliationFrequency is the number of blocks to wait between - // inactive reconiliations on each account. - InactiveReconciliationFrequency uint64 `json:"inactive_reconciliation_frequency"` - - // LogBlocks is a boolean indicating whether to log processed blocks. - LogBlocks bool `json:"log_blocks"` - - // LogTransactions is a boolean indicating whether to log processed transactions. - LogTransactions bool `json:"log_transactions"` - - // LogBalanceChanges is a boolean indicating whether to log all balance changes. - LogBalanceChanges bool `json:"log_balance_changes"` - - // LogReconciliations is a boolean indicating whether to log all reconciliations. - LogReconciliations bool `json:"log_reconciliations"` - - // IgnoreReconciliationError determines if block processing should halt on a reconciliation - // error. It can be beneficial to collect all reconciliation errors or silence - // reconciliation errors during development. - IgnoreReconciliationError bool `json:"ignore_reconciliation_error"` - - // ExemptAccounts is a path to a file listing all accounts to exempt from balance - // tracking and reconciliation. Look at the examples directory for an example of - // how to structure this file. - ExemptAccounts string `json:"exempt_accounts"` - - // BootstrapBalances is a path to a file used to bootstrap balances - // before starting syncing. If this value is populated after beginning syncing, - // it will be ignored. - BootstrapBalances string `json:"bootstrap_balances"` - - // HistoricalBalanceEnabled is a boolean that dictates how balance lookup is performed. - // When set to true, balances are looked up at the block where a balance - // change occurred instead of at the current block. Blockchains that do not support - // historical balance lookup should set this to false. - HistoricalBalanceEnabled *bool `json:"historical_balance_enabled,omitempty"` - - // InterestingAccounts is a path to a file listing all accounts to check on each block. Look - // at the examples directory for an example of how to structure this file. - InterestingAccounts string `json:"interesting_accounts"` - - // ReconciliationDisabled is a boolean that indicates reconciliation should not - // be attempted. When first testing an implementation, it can be useful to disable - // some of the more advanced checks to confirm syncing is working as expected. - ReconciliationDisabled bool `json:"reconciliation_disabled"` - - // InactiveDiscrepencySearchDisabled is a boolean indicating if a search - // should be performed to find any inactive reconciliation discrepencies. - // Note, a search will never be performed if historical balance lookup - // is disabled. - InactiveDiscrepencySearchDisabled bool `json:"inactive_discrepency_search_disabled"` - - // BalanceTrackingDisabled is a boolean that indicates balances calculation - // should not be attempted. When first testing an implemenation, it can be - // useful to just try to fetch all blocks before checking for balance - // consistency. - BalanceTrackingDisabled bool `json:"balance_tracking_disabled"` - - // CoinTrackingDisabled is a boolean that indicates coin (or UTXO) tracking - // should not be attempted. When first testing an implemenation, it can be - // useful to just try to fetch all blocks before checking for coin - // consistency. - CoinTrackingDisabled bool `json:"coin_tracking_disabled"` - - // StartIndex is the block height to start syncing from. If no StartIndex - // is provided, syncing will start from the last saved block. - // If no blocks have ever been synced, syncing will start from genesis. - StartIndex *int64 `json:"start_index,omitempty"` - - // EndCondition contains the conditions for the syncer to stop - EndConditions *DataEndConditions `json:"end_conditions,omitempty"` - - // StatusPort allows the caller to query a running check:data - // test to get stats about progress. This can be used instead - // of parsing logs to populate some sort of status dashboard. - StatusPort uint `json:"status_port,omitempty"` - - // ResultsOutputFile is the absolute filepath of where to save - // the results of a check:data run. - ResultsOutputFile string `json:"results_output_file"` - - // PruningDisabled is a bolean that indicates storage pruning should - // not be attempted. This should really only ever be set to true if you - // wish to use `start_index` at a later point to restart from some - // previously synced block. - PruningDisabled bool `json:"pruning_disabled"` -} - -// Configuration contains all configuration settings for running -// check:data or check:construction. -type Configuration struct { - // Network is the *types.NetworkIdentifier where transactions should - // be constructed and where blocks should be synced to monitor - // for broadcast success. - Network *types.NetworkIdentifier `json:"network"` - - // OnlineURL is the URL of a Rosetta API implementation in "online mode". - OnlineURL string `json:"online_url"` - - // DataDirectory is a folder used to store logs and any data used to perform validation. - DataDirectory string `json:"data_directory"` - - // HTTPTimeout is the timeout for a HTTP request in seconds. - HTTPTimeout uint64 `json:"http_timeout"` - - // MaxRetries is the number of times we will retry an HTTP request. If retry_elapsed_time - // is also populated, we may stop attempting retries early. - MaxRetries uint64 `json:"max_retries"` - - // RetryElapsedTime is the total time to spend retrying a HTTP request in seconds. - RetryElapsedTime uint64 `json:"retry_elapsed_time"` - - // MaxOnlineConnections is the maximum number of open connections that the online - // fetcher will open. - MaxOnlineConnections int `json:"max_online_connections"` - - // MaxSyncConcurrency is the maximum sync concurrency to use while syncing blocks. - // Sync concurrency is managed automatically by the `syncer` package. - MaxSyncConcurrency int64 `json:"max_sync_concurrency"` - - // TipDelay dictates how many seconds behind the current time is considered - // tip. If we are > TipDelay seconds from the last processed block, - // we are considered to be behind tip. - TipDelay int64 `json:"tip_delay"` - - // LogConfiguration determines if the configuration settings - // should be printed to the console when a file is loaded. - LogConfiguration bool `json:"log_configuration"` - - Construction *ConstructionConfiguration `json:"construction"` - Data *DataConfiguration `json:"data"` -} - func populateConstructionMissingFields( constructionConfig *ConstructionConfiguration, ) *ConstructionConfiguration { @@ -464,7 +171,8 @@ func assertConstructionConfiguration(ctx context.Context, config *ConstructionCo if len(config.ConstructorDSLFile) > 0 { compiledWorkflows, err := dsl.Parse(ctx, config.ConstructorDSLFile) if err != nil { - return fmt.Errorf("%s: compilation failed", types.PrintStruct(err)) + err.Log() + return fmt.Errorf("%w: compilation failed", err.Err) } config.Workflows = compiledWorkflows @@ -573,6 +281,28 @@ func assertConfiguration(ctx context.Context, config *Configuration) error { return nil } +// modifyFilePaths modifies a collection of filepaths in a *Configuration +// file to make them relative to the configuration file (this makes it a lot easier +// to store all config-related files in the same directory and to run the rosetta-cli +// from a different directory). +func modifyFilePaths(config *Configuration, fileDir string) { + if config.Data != nil { + if len(config.Data.BootstrapBalances) > 0 { + config.Data.BootstrapBalances = path.Join(fileDir, config.Data.BootstrapBalances) + } + + if len(config.Data.InterestingAccounts) > 0 { + config.Data.InterestingAccounts = path.Join(fileDir, config.Data.InterestingAccounts) + } + } + + if config.Construction != nil { + if len(config.Construction.ConstructorDSLFile) > 0 { + config.Construction.ConstructorDSLFile = path.Join(fileDir, config.Construction.ConstructorDSLFile) + } + } +} + // LoadConfiguration returns a parsed and asserted Configuration for running // tests. func LoadConfiguration(ctx context.Context, filePath string) (*Configuration, error) { @@ -583,6 +313,11 @@ func LoadConfiguration(ctx context.Context, filePath string) (*Configuration, er config := populateMissingFields(&configRaw) + // Get the configuration file directory so we can load all files + // relative to the location of the configuration file. + fileDir := path.Dir(filePath) + modifyFilePaths(config, fileDir) + if err := assertConfiguration(ctx, config); err != nil { return nil, fmt.Errorf("%w: invalid configuration", err) } diff --git a/configuration/configuration_test.go b/configuration/configuration_test.go index c89820b8..32365943 100644 --- a/configuration/configuration_test.go +++ b/configuration/configuration_test.go @@ -16,8 +16,8 @@ package configuration import ( "context" - "io/ioutil" - "os" + "os/exec" + "path" "testing" "github.com/coinbase/rosetta-sdk-go/constructor/job" @@ -178,7 +178,7 @@ func TestLoadConfiguration(t *testing.T) { "overwrite missing with DSL": { provided: &Configuration{ Construction: &ConstructionConfiguration{ - ConstructorDSLFile: "testdata/test.ros", + ConstructorDSLFile: "test.ros", }, Data: &DataConfiguration{}, }, @@ -192,7 +192,7 @@ func TestLoadConfiguration(t *testing.T) { BlockBroadcastLimit: DefaultBlockBroadcastLimit, StatusPort: DefaultStatusPort, Workflows: fakeWorkflows, - ConstructorDSLFile: "testdata/test.ros", + ConstructorDSLFile: "test.ros", } return cfg @@ -294,7 +294,7 @@ func TestLoadConfiguration(t *testing.T) { "non-existent dsl file": { provided: &Configuration{ Construction: &ConstructionConfiguration{ - ConstructorDSLFile: "test.ros", + ConstructorDSLFile: "blah.ros", }, }, err: true, @@ -313,24 +313,32 @@ func TestLoadConfiguration(t *testing.T) { for name, test := range tests { t.Run(name, func(t *testing.T) { - // Write configuration file to tempdir - tmpfile, err := ioutil.TempFile("", "test.json") + dir, err := utils.CreateTempDir() assert.NoError(t, err) - defer os.Remove(tmpfile.Name()) + defer utils.RemoveTempDir(dir) - err = utils.SerializeAndWrite(tmpfile.Name(), test.provided) + filePath := path.Join(dir, "test.json") + err = utils.SerializeAndWrite(filePath, test.provided) assert.NoError(t, err) + // Copy test.ros to temp dir + cmd := exec.Command("cp", "testdata/test.ros", path.Join(dir, "test.ros")) + assert.NoError(t, cmd.Run()) + // Check if expected fields populated - config, err := LoadConfiguration(context.Background(), tmpfile.Name()) + config, err := LoadConfiguration(context.Background(), filePath) if test.err { assert.Error(t, err) assert.Nil(t, config) } else { assert.NoError(t, err) + + // Ensure test.ros expected file path is right + if test.expected.Construction != nil && len(test.expected.Construction.ConstructorDSLFile) > 0 { + test.expected.Construction.ConstructorDSLFile = path.Join(dir, test.expected.Construction.ConstructorDSLFile) + } assert.Equal(t, test.expected, config) } - assert.NoError(t, tmpfile.Close()) }) } } diff --git a/configuration/types.go b/configuration/types.go new file mode 100644 index 00000000..0c45d60b --- /dev/null +++ b/configuration/types.go @@ -0,0 +1,302 @@ +package configuration + +import ( + "github.com/coinbase/rosetta-sdk-go/constructor/job" + "github.com/coinbase/rosetta-sdk-go/storage" + "github.com/coinbase/rosetta-sdk-go/types" +) + +// CheckDataEndCondition is a type of "successful" end +// for the "check:data" method. +type CheckDataEndCondition string + +const ( + // IndexEndCondition is used to indicate that the index end condition + // has been met. + IndexEndCondition CheckDataEndCondition = "Index End Condition" + + // DurationEndCondition is used to indicate that the duration + // end condition has been met. + DurationEndCondition CheckDataEndCondition = "Duration End Condition" + + // TipEndCondition is used to indicate that the tip end condition + // has been met. + TipEndCondition CheckDataEndCondition = "Tip End Condition" + + // ReconciliationCoverageEndCondition is used to indicate that the reconciliation + // coverage end condition has been met. + ReconciliationCoverageEndCondition CheckDataEndCondition = "Reconciliation Coverage End Condition" +) + +// Default Configuration Values +const ( + DefaultURL = "http://localhost:8080" + DefaultTimeout = 10 + DefaultMaxRetries = 5 + DefaultMaxOnlineConnections = 120 // most OS have a default limit of 128 + DefaultMaxOfflineConnections = 4 // we shouldn't need many connections for construction + DefaultMaxSyncConcurrency = 64 + DefaultActiveReconciliationConcurrency = 16 + DefaultInactiveReconciliationConcurrency = 4 + DefaultInactiveReconciliationFrequency = 250 + DefaultConfirmationDepth = 10 + DefaultStaleDepth = 30 + DefaultBroadcastLimit = 3 + DefaultTipDelay = 300 + DefaultBlockBroadcastLimit = 5 + DefaultStatusPort = 9090 + + // ETH Defaults + EthereumIDBlockchain = "Ethereum" + EthereumIDNetwork = "Ropsten" +) + +// Default Configuration Values +var ( + EthereumNetwork = &types.NetworkIdentifier{ + Blockchain: EthereumIDBlockchain, + Network: EthereumIDNetwork, + } +) + +// ConstructionConfiguration contains all configurations +// to run check:construction. +type ConstructionConfiguration struct { + // OfflineURL is the URL of a Rosetta API implementation in "offline mode". + OfflineURL string `json:"offline_url"` + + // MaxOffineConnections is the maximum number of open connections that the offline + // fetcher will open. + MaxOfflineConnections int `json:"max_offline_connections"` + + // StaleDepth is the number of blocks to wait before attempting + // to rebroadcast after not finding a transaction on-chain. + StaleDepth int64 `json:"stale_depth"` + + // BroadcastLimit is the number of times to attempt re-broadcast + // before giving up on a transaction broadcast. + BroadcastLimit int `json:"broadcast_limit"` + + // IgnoreBroadcastFailures determines if we should exit when there + // are broadcast failures (that surpass the BroadcastLimit). + IgnoreBroadcastFailures bool `json:"ignore_broadcast_failures"` + + // ClearBroadcasts indicates if all pending broadcasts should + // be removed from BroadcastStorage on restart. + ClearBroadcasts bool `json:"clear_broadcasts"` + + // BroadcastBehindTip indicates if we should broadcast transactions + // when we are behind tip (as defined by TipDelay). + BroadcastBehindTip bool `json:"broadcast_behind_tip"` + + // BlockBroadcastLimit is the number of transactions to attempt + // broadcast in a single block. When there are many pending + // broadcasts, it may make sense to limit the number of broadcasts. + BlockBroadcastLimit int `json:"block_broadcast_limit"` + + // RebroadcastAll indicates if all pending broadcasts should be + // rebroadcast from BroadcastStorage on restart. + RebroadcastAll bool `json:"rebroadcast_all"` + + // PrefundedAccounts is an array of prefunded accounts + // to use while testing. + PrefundedAccounts []*storage.PrefundedAccount `json:"prefunded_accounts,omitempty"` + + // Workflows are executed by the rosetta-cli to test + // certain construction flows. + Workflows []*job.Workflow `json:"workflows"` + + // ConstructorDSLFile is the path relative to the configuration file + // of a Rosetta Constructor DSL file (*.ros) + // that describes which Workflows to test. + // + // DSL Spec: https://github.com/coinbase/rosetta-sdk-go/tree/master/constructor/dsl + ConstructorDSLFile string `json:"constructor_dsl_file"` + + // EndConditions is a map of workflow:count that + // indicates how many of each workflow should be performed + // before check:construction should stop. For example, + // {"create_account": 5} indicates that 5 "create_account" + // workflows should be performed before stopping. + EndConditions map[string]int `json:"end_conditions,omitempty"` + + // StatusPort allows the caller to query a running check:construction + // test to get stats about progress. This can be used instead + // of parsing logs to populate some sort of status dashboard. + StatusPort uint `json:"status_port,omitempty"` + + // ResultsOutputFile is the absolute filepath of where to save + // the results of a check:construction run. + ResultsOutputFile string `json:"results_output_file,omitempty"` + + // Quiet is a boolean indicating if all request and response + // logging should be silenced. + Quiet bool `json:"quiet,omitempty"` +} + +// DataEndConditions contains all the conditions for the syncer to stop +// when running check:data. +type DataEndConditions struct { + // Index configures the syncer to stop once reaching a particular block height. + Index *int64 `json:"index,omitempty"` + + // Tip configures the syncer to stop once it reached the tip. + // Make sure to configure `tip_delay` if you use this end + // condition. + Tip *bool `json:"tip,omitempty"` + + // Duration configures the syncer to stop after running + // for Duration seconds. + Duration *uint64 `json:"duration,omitempty"` + + // ReconciliationCoverage configures the syncer to stop + // once it has reached tip AND some proportion of + // all addresses have been reconciled at an index >= + // to when tip was first reached. The range of inputs + // for this condition are [0.0, 1.0]. + ReconciliationCoverage *float64 `json:"reconciliation_coverage,omitempty"` +} + +// DataConfiguration contains all configurations to run check:data. +type DataConfiguration struct { + // ActiveReconciliationConcurrency is the concurrency to use while fetching accounts + // during active reconciliation. + ActiveReconciliationConcurrency uint64 `json:"active_reconciliation_concurrency"` + + // InactiveReconciliationConcurrency is the concurrency to use while fetching accounts + // during inactive reconciliation. + InactiveReconciliationConcurrency uint64 `json:"inactive_reconciliation_concurrency"` + + // InactiveReconciliationFrequency is the number of blocks to wait between + // inactive reconiliations on each account. + InactiveReconciliationFrequency uint64 `json:"inactive_reconciliation_frequency"` + + // LogBlocks is a boolean indicating whether to log processed blocks. + LogBlocks bool `json:"log_blocks"` + + // LogTransactions is a boolean indicating whether to log processed transactions. + LogTransactions bool `json:"log_transactions"` + + // LogBalanceChanges is a boolean indicating whether to log all balance changes. + LogBalanceChanges bool `json:"log_balance_changes"` + + // LogReconciliations is a boolean indicating whether to log all reconciliations. + LogReconciliations bool `json:"log_reconciliations"` + + // IgnoreReconciliationError determines if block processing should halt on a reconciliation + // error. It can be beneficial to collect all reconciliation errors or silence + // reconciliation errors during development. + IgnoreReconciliationError bool `json:"ignore_reconciliation_error"` + + // ExemptAccounts is a path relative to the configuration file + // to a file listing all accounts to exempt from balance + // tracking and reconciliation. Look at the examples directory for an example of + // how to structure this file. + ExemptAccounts string `json:"exempt_accounts"` + + // BootstrapBalances is a path relative to the configuration file to a file used + // to bootstrap balances before starting syncing. If this value is populated after + // beginning syncing, it will be ignored. + BootstrapBalances string `json:"bootstrap_balances"` + + // HistoricalBalanceEnabled is a boolean that dictates how balance lookup is performed. + // When set to true, balances are looked up at the block where a balance + // change occurred instead of at the current block. Blockchains that do not support + // historical balance lookup should set this to false. + HistoricalBalanceEnabled *bool `json:"historical_balance_enabled,omitempty"` + + // InterestingAccounts is a path to a file listing all accounts to check on each block. Look + // at the examples directory for an example of how to structure this file. + InterestingAccounts string `json:"interesting_accounts"` + + // ReconciliationDisabled is a boolean that indicates reconciliation should not + // be attempted. When first testing an implementation, it can be useful to disable + // some of the more advanced checks to confirm syncing is working as expected. + ReconciliationDisabled bool `json:"reconciliation_disabled"` + + // InactiveDiscrepencySearchDisabled is a boolean indicating if a search + // should be performed to find any inactive reconciliation discrepencies. + // Note, a search will never be performed if historical balance lookup + // is disabled. + InactiveDiscrepencySearchDisabled bool `json:"inactive_discrepency_search_disabled"` + + // BalanceTrackingDisabled is a boolean that indicates balances calculation + // should not be attempted. When first testing an implemenation, it can be + // useful to just try to fetch all blocks before checking for balance + // consistency. + BalanceTrackingDisabled bool `json:"balance_tracking_disabled"` + + // CoinTrackingDisabled is a boolean that indicates coin (or UTXO) tracking + // should not be attempted. When first testing an implemenation, it can be + // useful to just try to fetch all blocks before checking for coin + // consistency. + CoinTrackingDisabled bool `json:"coin_tracking_disabled"` + + // StartIndex is the block height to start syncing from. If no StartIndex + // is provided, syncing will start from the last saved block. + // If no blocks have ever been synced, syncing will start from genesis. + StartIndex *int64 `json:"start_index,omitempty"` + + // EndCondition contains the conditions for the syncer to stop + EndConditions *DataEndConditions `json:"end_conditions,omitempty"` + + // StatusPort allows the caller to query a running check:data + // test to get stats about progress. This can be used instead + // of parsing logs to populate some sort of status dashboard. + StatusPort uint `json:"status_port,omitempty"` + + // ResultsOutputFile is the absolute filepath of where to save + // the results of a check:data run. + ResultsOutputFile string `json:"results_output_file"` + + // PruningDisabled is a bolean that indicates storage pruning should + // not be attempted. This should really only ever be set to true if you + // wish to use `start_index` at a later point to restart from some + // previously synced block. + PruningDisabled bool `json:"pruning_disabled"` +} + +// Configuration contains all configuration settings for running +// check:data or check:construction. +type Configuration struct { + // Network is the *types.NetworkIdentifier where transactions should + // be constructed and where blocks should be synced to monitor + // for broadcast success. + Network *types.NetworkIdentifier `json:"network"` + + // OnlineURL is the URL of a Rosetta API implementation in "online mode". + OnlineURL string `json:"online_url"` + + // DataDirectory is a folder used to store logs and any data used to perform validation. + DataDirectory string `json:"data_directory"` + + // HTTPTimeout is the timeout for a HTTP request in seconds. + HTTPTimeout uint64 `json:"http_timeout"` + + // MaxRetries is the number of times we will retry an HTTP request. If retry_elapsed_time + // is also populated, we may stop attempting retries early. + MaxRetries uint64 `json:"max_retries"` + + // RetryElapsedTime is the total time to spend retrying a HTTP request in seconds. + RetryElapsedTime uint64 `json:"retry_elapsed_time"` + + // MaxOnlineConnections is the maximum number of open connections that the online + // fetcher will open. + MaxOnlineConnections int `json:"max_online_connections"` + + // MaxSyncConcurrency is the maximum sync concurrency to use while syncing blocks. + // Sync concurrency is managed automatically by the `syncer` package. + MaxSyncConcurrency int64 `json:"max_sync_concurrency"` + + // TipDelay dictates how many seconds behind the current time is considered + // tip. If we are > TipDelay seconds from the last processed block, + // we are considered to be behind tip. + TipDelay int64 `json:"tip_delay"` + + // LogConfiguration determines if the configuration settings + // should be printed to the console when a file is loaded. + LogConfiguration bool `json:"log_configuration"` + + Construction *ConstructionConfiguration `json:"construction"` + Data *DataConfiguration `json:"data"` +} diff --git a/examples/configuration/ethereum.json b/examples/configuration/ethereum.json index 7592bc93..295ca755 100644 --- a/examples/configuration/ethereum.json +++ b/examples/configuration/ethereum.json @@ -22,7 +22,7 @@ "broadcast_behind_tip": false, "block_broadcast_limit": 0, "rebroadcast_all": false, - "constructor_dsl_file": "examples/configuration/ethereum.ros", + "constructor_dsl_file": "ethereum.ros", "end_conditions": { "create_account": 10, "transfer": 10 From b8a24f591bbd9b38dd3cff5c892791202cec4083 Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Tue, 6 Oct 2020 09:08:28 -0700 Subject: [PATCH 088/376] Add license --- configuration/types.go | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/configuration/types.go b/configuration/types.go index 0c45d60b..f6622021 100644 --- a/configuration/types.go +++ b/configuration/types.go @@ -1,3 +1,17 @@ +// Copyright 2020 Coinbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package configuration import ( From af2b921620fea42f7a8103e85a8680311a7bd946 Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Tue, 6 Oct 2020 09:31:47 -0700 Subject: [PATCH 089/376] Update rosetta-sdk-go --- go.mod | 2 +- go.sum | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index 161b13dc..30e2ee0f 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/coinbase/rosetta-cli go 1.13 require ( - github.com/coinbase/rosetta-sdk-go v0.5.1-0.20201006040540-21c3125fd72d + github.com/coinbase/rosetta-sdk-go v0.5.1-0.20201006163035-c3ad17e072a1 github.com/fatih/color v1.9.0 github.com/jinzhu/copier v0.0.0-20190924061706-b57f9002281a github.com/olekukonko/tablewriter v0.0.2-0.20190409134802-7e037d187b0c diff --git a/go.sum b/go.sum index 7b3b21e1..68d7e4d8 100644 --- a/go.sum +++ b/go.sum @@ -166,6 +166,8 @@ github.com/coinbase/rosetta-sdk-go v0.5.0 h1:WdHJJ7RSUJ1sXwxVAoUb/owH3GYVEK2K3GU github.com/coinbase/rosetta-sdk-go v0.5.0/go.mod h1:QVVeKHWFNb0NyzEY06LxXMAylJkYa7n+Hk03pORr0ws= github.com/coinbase/rosetta-sdk-go v0.5.1-0.20201006040540-21c3125fd72d h1:GOfiiYEBWClqpXwRR2o7tKDrXUN3GKWv8Qvmi+kblmU= github.com/coinbase/rosetta-sdk-go v0.5.1-0.20201006040540-21c3125fd72d/go.mod h1:QVVeKHWFNb0NyzEY06LxXMAylJkYa7n+Hk03pORr0ws= +github.com/coinbase/rosetta-sdk-go v0.5.1-0.20201006163035-c3ad17e072a1 h1:feFtQluduAZKZBELfar7mm9XgkFTipct0q49iVww4Lw= +github.com/coinbase/rosetta-sdk-go v0.5.1-0.20201006163035-c3ad17e072a1/go.mod h1:QVVeKHWFNb0NyzEY06LxXMAylJkYa7n+Hk03pORr0ws= github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk= From d8afbed91041132dcce04ba73d4ea125230963cf Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Tue, 6 Oct 2020 10:27:35 -0700 Subject: [PATCH 090/376] Add comments to ethereum.ros --- examples/configuration/ethereum.ros | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/examples/configuration/ethereum.ros b/examples/configuration/ethereum.ros index 35a5d0c8..2f076f07 100644 --- a/examples/configuration/ethereum.ros +++ b/examples/configuration/ethereum.ros @@ -9,6 +9,10 @@ request_funds(1){ "create_limit":1 }); }, + + // Create a separate scenario to request funds so that + // the address we are using to request funds does not + // get rolled back if funds do not yet exist. request{ loaded_account = find_balance({ "account_identifier": {{random_account.account_identifier}}, @@ -28,6 +32,8 @@ create_account(1){ "network_identifier": {{network}}, "public_key": {{key.public_key}} }); + + // If the account is not saved, the key will be lost! save_account({ "account_identifier": {{account.account_identifier}}, "keypair": {{key}} @@ -45,10 +51,14 @@ transfer(10){ "currency": {{currency}} } }); - max_fee = set_variable("42000000000000"); + + // Set the recipient_amount as some value <= sender.balance-max_fee + max_fee = "84000000000000"; available_amount = {{sender.balance.value}} - {{max_fee}}; recipient_amount = random_number({"minimum": "1", "maximum": {{available_amount}}}); print_message({"recipient_amount":{{recipient_amount}}}); + + // Find recipient and construct operations sender_amount = 0 - {{recipient_amount}}; recipient = find_balance({ "not_account_identifier":[{{sender.account_identifier}}], @@ -84,19 +94,24 @@ transfer(10){ } return_funds(10){ + // TODO: add suggested fee dry run to ensure account fully cleared transfer{ transfer.network = {"network":"Ropsten", "blockchain":"Ethereum"}; currency = {"symbol":"ETH", "decimals":18}; - max_fee = "42000000000000"; + max_fee = "84000000000000"; sender = find_balance({ "minimum_balance":{ "value": {{max_fee}}, "currency": {{currency}} } }); + + // Set the recipient_amount as some sender.balance-max_fee available_amount = {{sender.balance.value}} - {{max_fee}}; print_message({"available_amount":{{available_amount}}}); sender_amount = 0 - {{available_amount}}; + + // Provide a static address as the recipient and construct operations faucet = {"address":"0xb41B39479a525AB69e38c701A713D98E3074252c"}; transfer.confirmation_depth = "1"; transfer.operations = [ From b99aaeb64318b40335e268bf3cf3f13f5d055545 Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Tue, 6 Oct 2020 10:43:35 -0700 Subject: [PATCH 091/376] Add extra options for reconciliation coverage --- configuration/types.go | 42 +++++++++++++++++++++++++++++++++++------- 1 file changed, 35 insertions(+), 7 deletions(-) diff --git a/configuration/types.go b/configuration/types.go index f6622021..e67a5ef2 100644 --- a/configuration/types.go +++ b/configuration/types.go @@ -148,8 +148,39 @@ type ConstructionConfiguration struct { Quiet bool `json:"quiet,omitempty"` } +// ReconciliationEndCondition is used to add conditions +// to reconciliation coverage for exiting `check:data`. +// +// If FromTip, Tip, Height, and AccountCount are not provided, +// `check:data` will halt as soon as coverage surpasses +// Coverage. +type ReconciliationEndCondition struct { + // Coverage is some value [0.0, 1.0] that represents + // the % of accounts reconciled. + Coverage float64 `json:"coverage"` + + // FromTip is a boolean indicating if reconciliation coverage + // should only be measured from tip (i.e. reconciliations + // performed at or after tip was reached). + FromTip *bool `json:"from_tip,omitempty"` + + // Tip is a boolean indicating that tip must be reached + // before reconciliation coverage is considered valid. + Tip *bool `json:"tip,omitempty"` + + // Height is an int64 indicating the height that must be + // reached before reconciliation coverage is considered valid. + Height *int64 `json:"height,omitempty"` + + // AccountCount is an int64 indicating the number of accounts + // that must be observed before reconciliation coverage is considered + // valid. + AccountCount *int64 `json:"account_count,omitempty"` +} + // DataEndConditions contains all the conditions for the syncer to stop -// when running check:data. +// when running check:data. If any one of these conditions is considered +// true, `check:data` will stop with success. type DataEndConditions struct { // Index configures the syncer to stop once reaching a particular block height. Index *int64 `json:"index,omitempty"` @@ -163,12 +194,9 @@ type DataEndConditions struct { // for Duration seconds. Duration *uint64 `json:"duration,omitempty"` - // ReconciliationCoverage configures the syncer to stop - // once it has reached tip AND some proportion of - // all addresses have been reconciled at an index >= - // to when tip was first reached. The range of inputs - // for this condition are [0.0, 1.0]. - ReconciliationCoverage *float64 `json:"reconciliation_coverage,omitempty"` + // Reconciliation configures the syncer to stop once it reaches + // some level of reconciliation coverage. + Reconciliation *ReconciliationEndCondition `json:"reconciliation,omitempty"` } // DataConfiguration contains all configurations to run check:data. From 401dc684a0b52205d0b3863c78ef3eca2e8fdc6e Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Tue, 6 Oct 2020 11:00:50 -0700 Subject: [PATCH 092/376] Pass existing tests --- configuration/configuration.go | 12 ++++- configuration/configuration_test.go | 75 +++++++++++++++++++++++++++-- configuration/types.go | 12 ++--- 3 files changed, 87 insertions(+), 12 deletions(-) diff --git a/configuration/configuration.go b/configuration/configuration.go index 8fa7c93e..0908f738 100644 --- a/configuration/configuration.go +++ b/configuration/configuration.go @@ -238,11 +238,21 @@ func assertDataConfiguration(config *DataConfiguration) error { } if config.EndConditions.ReconciliationCoverage != nil { - coverage := *config.EndConditions.ReconciliationCoverage + coverage := config.EndConditions.ReconciliationCoverage.Coverage if coverage < 0 || coverage > 1 { return fmt.Errorf("reconciliation coverage %f must be [0.0,1.0]", coverage) } + index := config.EndConditions.ReconciliationCoverage.Index + if index != nil && *index < 0 { + return fmt.Errorf("reconciliation coverage height %d must be >= 0", *index) + } + + accountCount := config.EndConditions.ReconciliationCoverage.AccountCount + if accountCount != nil && *accountCount < 0 { + return fmt.Errorf("reconciliation coverage account count %d must be >= 0", *accountCount) + } + if config.BalanceTrackingDisabled { return errors.New( "balance tracking must be enabled for reconciliation coverage end condition", diff --git a/configuration/configuration_test.go b/configuration/configuration_test.go index 32365943..84610f55 100644 --- a/configuration/configuration_test.go +++ b/configuration/configuration_test.go @@ -91,7 +91,9 @@ var ( StartIndex: &startIndex, StatusPort: 123, EndConditions: &DataEndConditions{ - ReconciliationCoverage: &goodCoverage, + ReconciliationCoverage: &ReconciliationCoverage{ + Coverage: goodCoverage, + }, }, }, } @@ -132,13 +134,19 @@ var ( invalidReconciliationCoverage = &Configuration{ Data: &DataConfiguration{ EndConditions: &DataEndConditions{ - ReconciliationCoverage: &badCoverage, + ReconciliationCoverage: &ReconciliationCoverage{ + Coverage: badCoverage, + }, }, }, } ) func TestLoadConfiguration(t *testing.T) { + var ( + goodAccountCount = int64(10) + badAccountCount = int64(-10) + ) var tests = map[string]struct { provided *Configuration expected *Configuration @@ -255,7 +263,9 @@ func TestLoadConfiguration(t *testing.T) { Data: &DataConfiguration{ ReconciliationDisabled: true, EndConditions: &DataEndConditions{ - ReconciliationCoverage: &goodCoverage, + ReconciliationCoverage: &ReconciliationCoverage{ + Coverage: goodCoverage, + }, }, }, }, @@ -266,7 +276,9 @@ func TestLoadConfiguration(t *testing.T) { Data: &DataConfiguration{ BalanceTrackingDisabled: true, EndConditions: &DataEndConditions{ - ReconciliationCoverage: &goodCoverage, + ReconciliationCoverage: &ReconciliationCoverage{ + Coverage: goodCoverage, + }, }, }, }, @@ -277,7 +289,60 @@ func TestLoadConfiguration(t *testing.T) { Data: &DataConfiguration{ IgnoreReconciliationError: true, EndConditions: &DataEndConditions{ - ReconciliationCoverage: &goodCoverage, + ReconciliationCoverage: &ReconciliationCoverage{ + Coverage: goodCoverage, + }, + }, + }, + }, + err: true, + }, + "valid reconciliation coverage (with account count)": { + provided: &Configuration{ + Data: &DataConfiguration{ + EndConditions: &DataEndConditions{ + ReconciliationCoverage: &ReconciliationCoverage{ + Coverage: goodCoverage, + AccountCount: &goodAccountCount, + Index: &goodAccountCount, + }, + }, + }, + }, + expected: func() *Configuration { + cfg := DefaultConfiguration() + cfg.Data.EndConditions = &DataEndConditions{ + ReconciliationCoverage: &ReconciliationCoverage{ + Coverage: goodCoverage, + AccountCount: &goodAccountCount, + Index: &goodAccountCount, + }, + } + + return cfg + }(), + }, + "invalid reconciliation coverage (with account count)": { + provided: &Configuration{ + Data: &DataConfiguration{ + EndConditions: &DataEndConditions{ + ReconciliationCoverage: &ReconciliationCoverage{ + Coverage: goodCoverage, + AccountCount: &badAccountCount, + }, + }, + }, + }, + err: true, + }, + "invalid reconciliation coverage (with index)": { + provided: &Configuration{ + Data: &DataConfiguration{ + EndConditions: &DataEndConditions{ + ReconciliationCoverage: &ReconciliationCoverage{ + Coverage: goodCoverage, + Index: &badAccountCount, + }, }, }, }, diff --git a/configuration/types.go b/configuration/types.go index e67a5ef2..08ae8816 100644 --- a/configuration/types.go +++ b/configuration/types.go @@ -148,13 +148,13 @@ type ConstructionConfiguration struct { Quiet bool `json:"quiet,omitempty"` } -// ReconciliationEndCondition is used to add conditions +// ReconciliationCoverage is used to add conditions // to reconciliation coverage for exiting `check:data`. // // If FromTip, Tip, Height, and AccountCount are not provided, // `check:data` will halt as soon as coverage surpasses // Coverage. -type ReconciliationEndCondition struct { +type ReconciliationCoverage struct { // Coverage is some value [0.0, 1.0] that represents // the % of accounts reconciled. Coverage float64 `json:"coverage"` @@ -168,9 +168,9 @@ type ReconciliationEndCondition struct { // before reconciliation coverage is considered valid. Tip *bool `json:"tip,omitempty"` - // Height is an int64 indicating the height that must be + // Index is an int64 indicating the height that must be // reached before reconciliation coverage is considered valid. - Height *int64 `json:"height,omitempty"` + Index *int64 `json:"height,omitempty"` // AccountCount is an int64 indicating the number of accounts // that must be observed before reconciliation coverage is considered @@ -194,9 +194,9 @@ type DataEndConditions struct { // for Duration seconds. Duration *uint64 `json:"duration,omitempty"` - // Reconciliation configures the syncer to stop once it reaches + // ReconciliationCoverage configures the syncer to stop once it reaches // some level of reconciliation coverage. - Reconciliation *ReconciliationEndCondition `json:"reconciliation,omitempty"` + ReconciliationCoverage *ReconciliationCoverage `json:"reconciliation_coverage,omitempty"` } // DataConfiguration contains all configurations to run check:data. From 5f468760b9572c38473a3dc826ad5df7c5291304 Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Tue, 6 Oct 2020 11:14:52 -0700 Subject: [PATCH 093/376] Add support for new reconciliation coverage --- configuration/types.go | 6 ++- pkg/tester/data.go | 100 +++++++++++++++++++++++++++++++---------- 2 files changed, 80 insertions(+), 26 deletions(-) diff --git a/configuration/types.go b/configuration/types.go index 08ae8816..902f52d9 100644 --- a/configuration/types.go +++ b/configuration/types.go @@ -150,6 +150,8 @@ type ConstructionConfiguration struct { // ReconciliationCoverage is used to add conditions // to reconciliation coverage for exiting `check:data`. +// All provided conditions must be satisfied before +// the end condition is considered satisfied. // // If FromTip, Tip, Height, and AccountCount are not provided, // `check:data` will halt as soon as coverage surpasses @@ -162,11 +164,11 @@ type ReconciliationCoverage struct { // FromTip is a boolean indicating if reconciliation coverage // should only be measured from tip (i.e. reconciliations // performed at or after tip was reached). - FromTip *bool `json:"from_tip,omitempty"` + FromTip bool `json:"from_tip,omitempty"` // Tip is a boolean indicating that tip must be reached // before reconciliation coverage is considered valid. - Tip *bool `json:"tip,omitempty"` + Tip bool `json:"tip,omitempty"` // Index is an int64 indicating the height that must be // reached before reconciliation coverage is considered valid. diff --git a/pkg/tester/data.go b/pkg/tester/data.go index d8065917..168ffcd6 100644 --- a/pkg/tester/data.go +++ b/pkg/tester/data.go @@ -388,13 +388,10 @@ func (t *DataTester) ServeHTTP(w http.ResponseWriter, r *http.Request) { // EndAtTipLoop runs a loop that evaluates end condition EndAtTip func (t *DataTester) EndAtTipLoop( ctx context.Context, - minReconciliationCoverage float64, ) { tc := time.NewTicker(EndAtTipCheckInterval) defer tc.Stop() - firstTipIndex := int64(-1) - for { select { case <-ctx.Done(): @@ -410,15 +407,7 @@ func (t *DataTester) EndAtTipLoop( continue } - // If we fall behind tip, we must reset the firstTipIndex. - if !atTip { - firstTipIndex = int64(-1) - continue - } - - // If minReconciliationCoverage is less than 0, - // we should just stop at tip. - if minReconciliationCoverage < 0 { + if atTip { t.endCondition = configuration.TipEndCondition t.endConditionDetail = fmt.Sprintf( "Tip: %d", @@ -427,26 +416,89 @@ func (t *DataTester) EndAtTipLoop( t.cancel() return } + } + } +} + +// EndReconciliationCoverage runs a loop that evaluates ReconciliationEndCondition +func (t *DataTester) EndReconciliationCoverage( + ctx context.Context, + reconciliationCoverage *configuration.ReconciliationCoverage, +) { + tc := time.NewTicker(EndAtTipCheckInterval) + defer tc.Stop() + + firstTipIndex := int64(-1) + + for { + select { + case <-ctx.Done(): + return + + case <-tc.C: + atTip, blockIdentifier, err := t.blockStorage.AtTip(ctx, t.config.TipDelay) + if err != nil { + log.Printf( + "%s: unable to evaluate syncer height or if at tip", + err.Error(), + ) + continue + } + + // Only check coverage if we are at tip. + if reconciliationCoverage.Tip || reconciliationCoverage.FromTip { + // If we fall behind tip, we must reset the firstTipIndex. + if !atTip { + firstTipIndex = int64(-1) + continue + } + + // Once at tip, we want to consider + // coverage. It is not feasible that we could + // get high reconciliation coverage at the tip + // block, so we take the range from when first + // at tip to the current block. + if firstTipIndex < 0 { + firstTipIndex = blockIdentifier.Index + } + } + + // Check if at required minimum index + if reconciliationCoverage.Index != nil && *reconciliationCoverage.Index < blockIdentifier.Index { + continue + } + + // Check if account count is above minimum index + if reconciliationCoverage.AccountCount != nil { + allAccounts, err := t.balanceStorage.GetAllAccountCurrency(ctx) + if err != nil { + log.Printf( + "%s: unable to get account count", + err.Error(), + ) + continue + } + + if int64(len(allAccounts)) < *reconciliationCoverage.AccountCount { + continue + } + } - // Once at tip, we want to consider - // coverage. It is not feasible that we could - // get high reconciliation coverage at the tip - // block, so we take the range from when first - // at tip to the current block. - if firstTipIndex < 0 { - firstTipIndex = blockIdentifier.Index + coverageIndex := int64(0) + if reconciliationCoverage.FromTip { + coverageIndex = firstTipIndex } - coverage, err := t.balanceStorage.ReconciliationCoverage(ctx, firstTipIndex) + coverage, err := t.balanceStorage.ReconciliationCoverage(ctx, coverageIndex) if err != nil { log.Printf( - "%s: unable to get reconciliations coverage", + "%s: unable to get reconciliation coverage", err.Error(), ) continue } - if coverage >= minReconciliationCoverage { + if coverage >= reconciliationCoverage.Coverage { t.endCondition = configuration.ReconciliationCoverageEndCondition t.endConditionDetail = fmt.Sprintf( "Coverage: %f%%", @@ -495,7 +547,7 @@ func (t *DataTester) WatchEndConditions( if endConds.Tip != nil && *endConds.Tip { // runs a go routine that ends when reaching tip - go t.EndAtTipLoop(ctx, -1) + go t.EndAtTipLoop(ctx) } if endConds.Duration != nil && *endConds.Duration != 0 { @@ -504,7 +556,7 @@ func (t *DataTester) WatchEndConditions( } if endConds.ReconciliationCoverage != nil { - go t.EndAtTipLoop(ctx, *endConds.ReconciliationCoverage) + go t.EndReconciliationCoverage(ctx, endConds.ReconciliationCoverage) } return nil From e393f1674fa7578d80a26294888027fd5e6871d7 Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Tue, 6 Oct 2020 11:22:02 -0700 Subject: [PATCH 094/376] nits --- examples/configuration/bitcoin.json | 5 ++++- examples/configuration/ethereum.json | 5 ++++- pkg/tester/data.go | 2 +- 3 files changed, 9 insertions(+), 3 deletions(-) diff --git a/examples/configuration/bitcoin.json b/examples/configuration/bitcoin.json index 9896ec5c..9afa31d5 100644 --- a/examples/configuration/bitcoin.json +++ b/examples/configuration/bitcoin.json @@ -373,7 +373,10 @@ "balance_tracking_disabled": false, "coin_tracking_disabled": false, "end_conditions": { - "reconciliation_coverage": 0.95 + "reconciliation_coverage": { + "coverage":0.95, + "from_tip": true + } }, "results_output_file": "" } diff --git a/examples/configuration/ethereum.json b/examples/configuration/ethereum.json index 295ca755..dcb3c13e 100644 --- a/examples/configuration/ethereum.json +++ b/examples/configuration/ethereum.json @@ -45,7 +45,10 @@ "balance_tracking_disabled": false, "coin_tracking_disabled": false, "end_conditions": { - "reconciliation_coverage": 0.95 + "reconciliation_coverage": { + "coverage":0.95, + "from_tip": true + } }, "results_output_file": "" } diff --git a/pkg/tester/data.go b/pkg/tester/data.go index 168ffcd6..09a9f0ec 100644 --- a/pkg/tester/data.go +++ b/pkg/tester/data.go @@ -421,7 +421,7 @@ func (t *DataTester) EndAtTipLoop( } // EndReconciliationCoverage runs a loop that evaluates ReconciliationEndCondition -func (t *DataTester) EndReconciliationCoverage( +func (t *DataTester) EndReconciliationCoverage( // nolint:gocognit ctx context.Context, reconciliationCoverage *configuration.ReconciliationCoverage, ) { From 94df856604e6496cc0c3f919175d19cbb8e3ad89 Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Tue, 6 Oct 2020 11:35:00 -0700 Subject: [PATCH 095/376] Add note about DSL in README --- README.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/README.md b/README.md index d559e382..08e1329e 100644 --- a/README.md +++ b/README.md @@ -94,6 +94,11 @@ The new Construction API testing framework (first released in `rosetta-cli@v0.5. a new design pattern to allow for complex transaction construction orchestration. You can read more about the design goals [here](https://community.rosetta-api.org/t/feedback-request-automated-construction-api-testing-improvements/146). +Most teams write their Construction API tests using the +[Rosetta Constructor DSL](https://github.com/coinbase/rosetta-sdk-go/tree/master/constructor/dsl). +You can find an example of a file written in this DSL in the examples +folder ([here](https://github.com/coinbase/rosetta-cli/blob/master/examples/configuration/ethereum.ros)). + ##### Terminology When first learning about a new topic, it is often useful to understand the hierarchy of concerns. In the automated Construction API tester, this From 9d88e8feef2eb6baef47837b030924b31e878228 Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Tue, 6 Oct 2020 11:47:49 -0700 Subject: [PATCH 096/376] Update rosetta-sdk-go --- go.mod | 2 +- go.sum | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index 30e2ee0f..c24e54fc 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/coinbase/rosetta-cli go 1.13 require ( - github.com/coinbase/rosetta-sdk-go v0.5.1-0.20201006163035-c3ad17e072a1 + github.com/coinbase/rosetta-sdk-go v0.5.1 github.com/fatih/color v1.9.0 github.com/jinzhu/copier v0.0.0-20190924061706-b57f9002281a github.com/olekukonko/tablewriter v0.0.2-0.20190409134802-7e037d187b0c diff --git a/go.sum b/go.sum index 68d7e4d8..5c2d3448 100644 --- a/go.sum +++ b/go.sum @@ -168,6 +168,8 @@ github.com/coinbase/rosetta-sdk-go v0.5.1-0.20201006040540-21c3125fd72d h1:GOfii github.com/coinbase/rosetta-sdk-go v0.5.1-0.20201006040540-21c3125fd72d/go.mod h1:QVVeKHWFNb0NyzEY06LxXMAylJkYa7n+Hk03pORr0ws= github.com/coinbase/rosetta-sdk-go v0.5.1-0.20201006163035-c3ad17e072a1 h1:feFtQluduAZKZBELfar7mm9XgkFTipct0q49iVww4Lw= github.com/coinbase/rosetta-sdk-go v0.5.1-0.20201006163035-c3ad17e072a1/go.mod h1:QVVeKHWFNb0NyzEY06LxXMAylJkYa7n+Hk03pORr0ws= +github.com/coinbase/rosetta-sdk-go v0.5.1 h1:UGuEt/nMWHvnLmWZqJdoQP+HhWL5/6K04rjhPHhmS2M= +github.com/coinbase/rosetta-sdk-go v0.5.1/go.mod h1:QVVeKHWFNb0NyzEY06LxXMAylJkYa7n+Hk03pORr0ws= github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk= From 2b61b0a8a63f806b7c1e309fce15a9af2c921519 Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Tue, 6 Oct 2020 12:07:00 -0700 Subject: [PATCH 097/376] Fix asserter log error --- go.mod | 2 +- go.sum | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index c24e54fc..4b8b86df 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/coinbase/rosetta-cli go 1.13 require ( - github.com/coinbase/rosetta-sdk-go v0.5.1 + github.com/coinbase/rosetta-sdk-go v0.5.2-0.20201006190307-bf4606611446 github.com/fatih/color v1.9.0 github.com/jinzhu/copier v0.0.0-20190924061706-b57f9002281a github.com/olekukonko/tablewriter v0.0.2-0.20190409134802-7e037d187b0c diff --git a/go.sum b/go.sum index 5c2d3448..02c8019c 100644 --- a/go.sum +++ b/go.sum @@ -170,6 +170,8 @@ github.com/coinbase/rosetta-sdk-go v0.5.1-0.20201006163035-c3ad17e072a1 h1:feFtQ github.com/coinbase/rosetta-sdk-go v0.5.1-0.20201006163035-c3ad17e072a1/go.mod h1:QVVeKHWFNb0NyzEY06LxXMAylJkYa7n+Hk03pORr0ws= github.com/coinbase/rosetta-sdk-go v0.5.1 h1:UGuEt/nMWHvnLmWZqJdoQP+HhWL5/6K04rjhPHhmS2M= github.com/coinbase/rosetta-sdk-go v0.5.1/go.mod h1:QVVeKHWFNb0NyzEY06LxXMAylJkYa7n+Hk03pORr0ws= +github.com/coinbase/rosetta-sdk-go v0.5.2-0.20201006190307-bf4606611446 h1:KbefWZdPOb0yBl5DGV/FfJ95OJMiHxdxiJGLY+EjlT0= +github.com/coinbase/rosetta-sdk-go v0.5.2-0.20201006190307-bf4606611446/go.mod h1:QVVeKHWFNb0NyzEY06LxXMAylJkYa7n+Hk03pORr0ws= github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk= From 7b130659bc3cf3da52bb508975ff7a680326a243 Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Thu, 8 Oct 2020 15:07:36 -0700 Subject: [PATCH 098/376] Fix panic --- configuration/types.go | 2 +- pkg/tester/data.go | 11 +++++++++-- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/configuration/types.go b/configuration/types.go index 902f52d9..d0ec97c8 100644 --- a/configuration/types.go +++ b/configuration/types.go @@ -172,7 +172,7 @@ type ReconciliationCoverage struct { // Index is an int64 indicating the height that must be // reached before reconciliation coverage is considered valid. - Index *int64 `json:"height,omitempty"` + Index *int64 `json:"index,omitempty"` // AccountCount is an int64 indicating the number of accounts // that must be observed before reconciliation coverage is considered diff --git a/pkg/tester/data.go b/pkg/tester/data.go index 09a9f0ec..23d0751d 100644 --- a/pkg/tester/data.go +++ b/pkg/tester/data.go @@ -436,7 +436,10 @@ func (t *DataTester) EndReconciliationCoverage( // nolint:gocognit return case <-tc.C: - atTip, blockIdentifier, err := t.blockStorage.AtTip(ctx, t.config.TipDelay) + headBlock, err := t.blockStorage.GetBlock(ctx, nil) + if errors.Is(err, storage.ErrHeadBlockNotFound) { + continue + } if err != nil { log.Printf( "%s: unable to evaluate syncer height or if at tip", @@ -445,7 +448,10 @@ func (t *DataTester) EndReconciliationCoverage( // nolint:gocognit continue } - // Only check coverage if we are at tip. + blockIdentifier := headBlock.BlockIdentifier + atTip := utils.AtTip(t.config.TipDelay, headBlock.Timestamp) + + // Check if we are at tip and set tip height if fromTip is true. if reconciliationCoverage.Tip || reconciliationCoverage.FromTip { // If we fall behind tip, we must reset the firstTipIndex. if !atTip { @@ -464,6 +470,7 @@ func (t *DataTester) EndReconciliationCoverage( // nolint:gocognit } // Check if at required minimum index + // TODO: blockIdentifer is nil if not at tip if reconciliationCoverage.Index != nil && *reconciliationCoverage.Index < blockIdentifier.Index { continue } From 0ea119c131bfe9590a2e2817fcbd2a91db8fe800 Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Thu, 8 Oct 2020 15:08:10 -0700 Subject: [PATCH 099/376] nits --- pkg/tester/data.go | 1 - 1 file changed, 1 deletion(-) diff --git a/pkg/tester/data.go b/pkg/tester/data.go index 23d0751d..a790b132 100644 --- a/pkg/tester/data.go +++ b/pkg/tester/data.go @@ -470,7 +470,6 @@ func (t *DataTester) EndReconciliationCoverage( // nolint:gocognit } // Check if at required minimum index - // TODO: blockIdentifer is nil if not at tip if reconciliationCoverage.Index != nil && *reconciliationCoverage.Index < blockIdentifier.Index { continue } From dd9267767b4931fc50fe26550c000618e77a46d9 Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Thu, 8 Oct 2020 15:12:41 -0700 Subject: [PATCH 100/376] Update version --- cmd/root.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/root.go b/cmd/root.go index 7e7e4c7c..f0f9ee09 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -217,6 +217,6 @@ var versionCmd = &cobra.Command{ Use: "version", Short: "Print rosetta-cli version", Run: func(cmd *cobra.Command, args []string) { - fmt.Println("v0.5.8") + fmt.Println("v0.5.9") }, } From 1b8b7da29e1cdb4daad6b2869937a1d09fcaa12e Mon Sep 17 00:00:00 2001 From: Daniel Van Der Maden Date: Mon, 12 Oct 2020 21:13:49 -0700 Subject: [PATCH 101/376] Call online node for Coordinator's Metadata --- pkg/processor/coordinator_helper.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/processor/coordinator_helper.go b/pkg/processor/coordinator_helper.go index a252e280..10dd75cc 100644 --- a/pkg/processor/coordinator_helper.go +++ b/pkg/processor/coordinator_helper.go @@ -204,7 +204,7 @@ func (c *CoordinatorHelper) Metadata( arg{argMetadata, metadataRequest}, arg{argPublicKeys, publicKeys}, ) - metadata, suggestedFee, fetchErr := c.offlineFetcher.ConstructionMetadata( + metadata, suggestedFee, fetchErr := c.onlineFetcher.ConstructionMetadata( ctx, networkIdentifier, metadataRequest, From e9204fafb0d28af040005b9b20f9a6d8e27b9b87 Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Mon, 12 Oct 2020 22:45:45 -0700 Subject: [PATCH 102/376] Update version --- cmd/root.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/root.go b/cmd/root.go index f0f9ee09..1b8cbc75 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -217,6 +217,6 @@ var versionCmd = &cobra.Command{ Use: "version", Short: "Print rosetta-cli version", Run: func(cmd *cobra.Command, args []string) { - fmt.Println("v0.5.9") + fmt.Println("v0.5.10") }, } From f771ea2fb542aeec059ef56f6ddc6f56caf45041 Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Tue, 13 Oct 2020 15:44:38 -0700 Subject: [PATCH 103/376] Update rosetta-sdk-go --- go.mod | 2 +- go.sum | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index 4b8b86df..dd9de391 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/coinbase/rosetta-cli go 1.13 require ( - github.com/coinbase/rosetta-sdk-go v0.5.2-0.20201006190307-bf4606611446 + github.com/coinbase/rosetta-sdk-go v0.5.2 github.com/fatih/color v1.9.0 github.com/jinzhu/copier v0.0.0-20190924061706-b57f9002281a github.com/olekukonko/tablewriter v0.0.2-0.20190409134802-7e037d187b0c diff --git a/go.sum b/go.sum index 02c8019c..db9502e8 100644 --- a/go.sum +++ b/go.sum @@ -172,6 +172,8 @@ github.com/coinbase/rosetta-sdk-go v0.5.1 h1:UGuEt/nMWHvnLmWZqJdoQP+HhWL5/6K04rj github.com/coinbase/rosetta-sdk-go v0.5.1/go.mod h1:QVVeKHWFNb0NyzEY06LxXMAylJkYa7n+Hk03pORr0ws= github.com/coinbase/rosetta-sdk-go v0.5.2-0.20201006190307-bf4606611446 h1:KbefWZdPOb0yBl5DGV/FfJ95OJMiHxdxiJGLY+EjlT0= github.com/coinbase/rosetta-sdk-go v0.5.2-0.20201006190307-bf4606611446/go.mod h1:QVVeKHWFNb0NyzEY06LxXMAylJkYa7n+Hk03pORr0ws= +github.com/coinbase/rosetta-sdk-go v0.5.2 h1:G91QWW0wrqmMhW2vJ4LP9P3gVukZs7df2Gf6FF1uff0= +github.com/coinbase/rosetta-sdk-go v0.5.2/go.mod h1:QVVeKHWFNb0NyzEY06LxXMAylJkYa7n+Hk03pORr0ws= github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk= From 2f8448eb88315b88ba81f849ed0a6d331c3fefc6 Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Tue, 13 Oct 2020 15:46:24 -0700 Subject: [PATCH 104/376] Update version --- cmd/root.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/root.go b/cmd/root.go index 1b8cbc75..19a9b846 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -217,6 +217,6 @@ var versionCmd = &cobra.Command{ Use: "version", Short: "Print rosetta-cli version", Run: func(cmd *cobra.Command, args []string) { - fmt.Println("v0.5.10") + fmt.Println("v0.5.11") }, } From 8ff28f6283465646e70b19a85f5c70538531333f Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Tue, 13 Oct 2020 15:48:32 -0700 Subject: [PATCH 105/376] Replace node references with live --- pkg/logger/logger.go | 14 +++++++------- pkg/processor/reconciler_handler.go | 10 +++++----- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/pkg/logger/logger.go b/pkg/logger/logger.go index 74f0f9cf..5074fe2c 100644 --- a/pkg/logger/logger.go +++ b/pkg/logger/logger.go @@ -404,27 +404,27 @@ func (l *Logger) ReconcileFailureStream( account *types.AccountIdentifier, currency *types.Currency, computedBalance string, - nodeBalance string, + liveBalance string, block *types.BlockIdentifier, ) error { // Always print out reconciliation failures if reconciliationType == reconciler.InactiveReconciliation { color.Yellow( - "Missing balance-changing operation detected for %s computed balance: %s%s node balance: %s%s", + "Missing balance-changing operation detected for %s computed: %s%s live: %s%s", types.AccountString(account), computedBalance, currency.Symbol, - nodeBalance, + liveBalance, currency.Symbol, ) } else { color.Yellow( - "Reconciliation failed for %s at %d computed: %s%s node: %s%s", + "Reconciliation failed for %s at %d computed: %s%s live: %s%s", types.AccountString(account), block.Index, computedBalance, currency.Symbol, - nodeBalance, + liveBalance, currency.Symbol, ) } @@ -445,14 +445,14 @@ func (l *Logger) ReconcileFailureStream( defer closeFile(f) _, err = f.WriteString(fmt.Sprintf( - "Type:%s Account: %s Currency: %s Block: %s:%d computed: %s node: %s\n", + "Type:%s Account: %s Currency: %s Block: %s:%d computed: %s live: %s\n", reconciliationType, types.AccountString(account), types.CurrencyString(currency), block.Hash, block.Index, computedBalance, - nodeBalance, + liveBalance, )) if err != nil { return err diff --git a/pkg/processor/reconciler_handler.go b/pkg/processor/reconciler_handler.go index b1a5e655..53c0fe60 100644 --- a/pkg/processor/reconciler_handler.go +++ b/pkg/processor/reconciler_handler.go @@ -66,7 +66,7 @@ func (h *ReconcilerHandler) ReconciliationFailed( account *types.AccountIdentifier, currency *types.Currency, computedBalance string, - nodeBalance string, + liveBalance string, block *types.BlockIdentifier, ) error { err := h.logger.ReconcileFailureStream( @@ -75,7 +75,7 @@ func (h *ReconcilerHandler) ReconciliationFailed( account, currency, computedBalance, - nodeBalance, + liveBalance, block, ) if err != nil { @@ -98,7 +98,7 @@ func (h *ReconcilerHandler) ReconciliationFailed( block.Index, computedBalance, currency.Symbol, - nodeBalance, + liveBalance, currency.Symbol, ) } @@ -112,7 +112,7 @@ func (h *ReconcilerHandler) ReconciliationFailed( block.Index, computedBalance, currency.Symbol, - nodeBalance, + liveBalance, currency.Symbol, ) } @@ -128,7 +128,7 @@ func (h *ReconcilerHandler) ReconciliationExempt( account *types.AccountIdentifier, currency *types.Currency, computedBalance string, - nodeBalance string, + liveBalance string, block *types.BlockIdentifier, exemption *types.BalanceExemption, ) error { From 2741c6087eb630a31f6e76abcd0043e2983e26e2 Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Wed, 14 Oct 2020 16:00:35 -0700 Subject: [PATCH 106/376] Allow for outputing block profiles --- cmd/root.go | 100 ++++++++++++++++++++++++++++++++++------------------ 1 file changed, 65 insertions(+), 35 deletions(-) diff --git a/cmd/root.go b/cmd/root.go index 19a9b846..4b9687e3 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -41,6 +41,7 @@ var ( configurationFile string cpuProfile string memProfile string + blockProfile string // Config is the populated *configuration.Configuration from // the configurationFile. If none is provided, this is set @@ -54,9 +55,13 @@ var ( // determining the error message to show on exit much more easy. SignalReceived = false - // profileCleanup is called after the root command is executed to + // cpuProfileCleanup is called after the root command is executed to // cleanup a running cpu profile. - profileCleanup func() + cpuProfileCleanup func() + + // blockProfileCleanup is called after the root command is executed to + // cleanup a running block profile. + blockProfileCleanup func() ) // rootPreRun is executed before the root command runs and sets up cpu @@ -64,56 +69,75 @@ var ( // // Bassed on https://golang.org/pkg/runtime/pprof/#hdr-Profiling_a_Go_program func rootPreRun(*cobra.Command, []string) error { - if cpuProfile == "" { - return nil - } + if cpuProfile != "" { + f, err := os.Create(cpuProfile) + if err != nil { + return fmt.Errorf("%w: unable to create CPU profile file", err) + } + if err := pprof.StartCPUProfile(f); err != nil { + if err := f.Close(); err != nil { + log.Printf("error while closing cpu profile file: %v\n", err) + } + return err + } - f, err := os.Create(cpuProfile) - if err != nil { - return fmt.Errorf("%w: unable to create CPU profile file", err) - } - if err := pprof.StartCPUProfile(f); err != nil { - if err := f.Close(); err != nil { - log.Printf("error while closing cpu profile file: %v\n", err) + cpuProfileCleanup = func() { + pprof.StopCPUProfile() + if err := f.Close(); err != nil { + log.Printf("error while closing cpu profile file: %v\n", err) + } } - return err } - profileCleanup = func() { - pprof.StopCPUProfile() - if err := f.Close(); err != nil { - log.Printf("error while closing cpu profile file: %v\n", err) + if blockProfile != "" { + runtime.SetBlockProfileRate(1) + f, err := os.Create(blockProfile) + if err != nil { + return fmt.Errorf("%w: unable to create block profile file", err) + } + + p := pprof.Lookup("block") + blockProfileCleanup = func() { + if err := p.WriteTo(f, 0); err != nil { + log.Printf("error while writing block profile file: %v\n", err) + } + if err := f.Close(); err != nil { + log.Printf("error while closing block profile file: %v\n", err) + } } } + return nil } // rootPostRun is executed after the root command runs and performs memory // profiling. func rootPostRun() { - if profileCleanup != nil { - profileCleanup() + if cpuProfileCleanup != nil { + cpuProfileCleanup() } - if memProfile == "" { - return + if blockProfileCleanup != nil { + blockProfileCleanup() } - f, err := os.Create(memProfile) - if err != nil { - log.Printf("error while creating mem-profile file: %v", err) - return - } - - defer func() { - if err := f.Close(); err != nil { - log.Printf("error while closing mem-profile file: %v", err) + if memProfile != "" { + f, err := os.Create(memProfile) + if err != nil { + log.Printf("error while creating mem-profile file: %v", err) + return } - }() - runtime.GC() - if err := pprof.WriteHeapProfile(f); err != nil { - log.Printf("error while writing heap profile: %v", err) + defer func() { + if err := f.Close(); err != nil { + log.Printf("error while closing mem-profile file: %v", err) + } + }() + + runtime.GC() + if err := pprof.WriteHeapProfile(f); err != nil { + log.Printf("error while writing heap profile: %v", err) + } } } @@ -151,6 +175,12 @@ default values.`, "", `Save the pprof mem profile in the specified file`, ) + rootFlags.StringVar( + &blockProfile, + "block-profile", + "", + `Save the pprof block profile in the specified file`, + ) rootCmd.AddCommand(versionCmd) // Configuration Commands @@ -217,6 +247,6 @@ var versionCmd = &cobra.Command{ Use: "version", Short: "Print rosetta-cli version", Run: func(cmd *cobra.Command, args []string) { - fmt.Println("v0.5.11") + fmt.Println("v0.5.12") }, } From 317536673bad8f67edfbb987f2ab445909563f94 Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Wed, 14 Oct 2020 17:30:00 -0700 Subject: [PATCH 107/376] Use buffered channels --- go.mod | 2 +- go.sum | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index dd9de391..041762d7 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/coinbase/rosetta-cli go 1.13 require ( - github.com/coinbase/rosetta-sdk-go v0.5.2 + github.com/coinbase/rosetta-sdk-go v0.5.3-0.20201015002757-978c228143a3 github.com/fatih/color v1.9.0 github.com/jinzhu/copier v0.0.0-20190924061706-b57f9002281a github.com/olekukonko/tablewriter v0.0.2-0.20190409134802-7e037d187b0c diff --git a/go.sum b/go.sum index db9502e8..6b1dbdac 100644 --- a/go.sum +++ b/go.sum @@ -174,6 +174,8 @@ github.com/coinbase/rosetta-sdk-go v0.5.2-0.20201006190307-bf4606611446 h1:KbefW github.com/coinbase/rosetta-sdk-go v0.5.2-0.20201006190307-bf4606611446/go.mod h1:QVVeKHWFNb0NyzEY06LxXMAylJkYa7n+Hk03pORr0ws= github.com/coinbase/rosetta-sdk-go v0.5.2 h1:G91QWW0wrqmMhW2vJ4LP9P3gVukZs7df2Gf6FF1uff0= github.com/coinbase/rosetta-sdk-go v0.5.2/go.mod h1:QVVeKHWFNb0NyzEY06LxXMAylJkYa7n+Hk03pORr0ws= +github.com/coinbase/rosetta-sdk-go v0.5.3-0.20201015002757-978c228143a3 h1:p8p0dVZDh/Zs35tzQq3nFftgFbphNvSzZ6dzLT31F7I= +github.com/coinbase/rosetta-sdk-go v0.5.3-0.20201015002757-978c228143a3/go.mod h1:QVVeKHWFNb0NyzEY06LxXMAylJkYa7n+Hk03pORr0ws= github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk= From 93fd58187afe2825e9441f2124303379216282fb Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Wed, 14 Oct 2020 17:41:54 -0700 Subject: [PATCH 108/376] Ensure we only exit with reconciliation coverage when reconciler is caught up. --- pkg/tester/data.go | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/pkg/tester/data.go b/pkg/tester/data.go index a790b132..d8b8e9bc 100644 --- a/pkg/tester/data.go +++ b/pkg/tester/data.go @@ -469,8 +469,27 @@ func (t *DataTester) EndReconciliationCoverage( // nolint:gocognit } } + // minIndex is the greater of firstTipIndex + // and reconciliationCoverage.Index + minIndex := firstTipIndex + // Check if at required minimum index - if reconciliationCoverage.Index != nil && *reconciliationCoverage.Index < blockIdentifier.Index { + if reconciliationCoverage.Index != nil { + if *reconciliationCoverage.Index < blockIdentifier.Index { + continue + } + + // Override the firstTipIndex if reconciliationCoverage.Index + // is greater + if *reconciliationCoverage.Index > minIndex { + minIndex = *reconciliationCoverage.Index + } + } + + // Check if all accounts reconciled at index (+1). If last index reconciled + // is less than the minimum allowed index but the QueueSize is 0, then + // we consider the reconciler to be caught up. + if t.reconciler.LastIndexReconciled() <= minIndex && t.reconciler.QueueSize() > 0 { continue } From 14ff64658bf24c6b608d186d355c59f42e53a1ff Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 15 Oct 2020 05:21:32 +0000 Subject: [PATCH 109/376] Bump github.com/spf13/cobra from 1.0.0 to 1.1.0 Bumps [github.com/spf13/cobra](https://github.com/spf13/cobra) from 1.0.0 to 1.1.0. - [Release notes](https://github.com/spf13/cobra/releases) - [Changelog](https://github.com/spf13/cobra/blob/master/CHANGELOG.md) - [Commits](https://github.com/spf13/cobra/compare/v1.0.0...v1.1.0) Signed-off-by: dependabot[bot] --- go.mod | 5 +- go.sum | 289 +++++++++++++++++++++++++++------------------------------ 2 files changed, 139 insertions(+), 155 deletions(-) diff --git a/go.mod b/go.mod index 041762d7..59790fac 100644 --- a/go.mod +++ b/go.mod @@ -5,11 +5,8 @@ go 1.13 require ( github.com/coinbase/rosetta-sdk-go v0.5.3-0.20201015002757-978c228143a3 github.com/fatih/color v1.9.0 - github.com/jinzhu/copier v0.0.0-20190924061706-b57f9002281a github.com/olekukonko/tablewriter v0.0.2-0.20190409134802-7e037d187b0c - github.com/spf13/cobra v1.0.0 - github.com/spf13/pflag v1.0.5 // indirect + github.com/spf13/cobra v1.1.0 github.com/stretchr/testify v1.6.1 golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208 - src.techknowlogick.com/xgo v1.1.1-0.20200814033943-12cf2e8194ca // indirect ) diff --git a/go.sum b/go.sum index 6b1dbdac..7a6f0a75 100644 --- a/go.sum +++ b/go.sum @@ -1,4 +1,16 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= +cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU= +cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= +cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc= +cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0= +cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= +cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= +cloud.google.com/go/firestore v1.1.0/go.mod h1:ulACoGHTpvq5r8rxGJ4ddJZBZqakUQqClKRT5SZwBmk= +cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= +cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= +dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= github.com/Azure/azure-pipeline-go v0.2.1/go.mod h1:UGSo8XybXnIGZ3epmeBw7Jdz+HiUVpqIlpz/HKHylF4= github.com/Azure/azure-pipeline-go v0.2.2/go.mod h1:4rQ/NZncSvGqNkkOsNpOU1tgoNuIlp9AfUH5G1tvCHc= github.com/Azure/azure-storage-blob-go v0.7.0/go.mod h1:f9YQKtsG1nMisotuTPpO0tjNuEjKRYAcJU8/ydDI++4= @@ -34,10 +46,15 @@ github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk5 github.com/allegro/bigcache v1.2.1-0.20190218064605-e24eb225f156/go.mod h1:Cb/ax3seSYIx7SuZdm2G2xzfwmv3TPSk2ucNfQESPXM= github.com/aristanetworks/goarista v0.0.0-20170210015632-ea17b1a17847 h1:rtI0fD4oG/8eVokGVPYJEW1F88p1ZNgXiEIs9thEE4A= github.com/aristanetworks/goarista v0.0.0-20170210015632-ea17b1a17847/go.mod h1:D/tb0zPVXnP7fmsLZjtdUhSsumbK/ij54UXjjVgMGxQ= +github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= +github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= +github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= github.com/aws/aws-sdk-go v1.25.48/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= +github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= +github.com/bketelsen/crypt v0.0.3-0.20200106085610-5cbc8cc4026c/go.mod h1:MKsuJmJgSg28kpZDP6UIiPt0e0Oz0kqKNGyRaWEPv84= github.com/btcsuite/btcd v0.0.0-20171128150713-2e60448ffcc6/go.mod h1:Dmm/EzmjnCiweXmzRIAiUWCInVmPgjkzgv5k4tVyXiQ= github.com/btcsuite/btcd v0.20.1-beta h1:Ik4hyJqN8Jfyv3S4AGBOmyouMsYE3EdYODkMbQjwPGw= github.com/btcsuite/btcd v0.20.1-beta/go.mod h1:wVuoA8VJLEcwgqHBwHmzLRazpKxTv13Px/pDuV7OomQ= @@ -63,123 +80,14 @@ github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+ github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cloudflare/cloudflare-go v0.10.2-0.20190916151808-a80f83b9add9/go.mod h1:1MxXX1Ux4x6mqPmjkUgTP1CdXIBXKX7T+Jk9Gxrmx+U= -github.com/coinbase/rosetta-sdk-go v0.3.4 h1:jWKgajozco/T0FNnZb2TqBsmsUoF6ZuCLnUJkEE+vNg= -github.com/coinbase/rosetta-sdk-go v0.3.5-0.20200824221853-d7b1fe2f9239 h1:eP88DoIkcbMG5zHja3lUh4F2c+9am1T0zk/AChy1Vpo= -github.com/coinbase/rosetta-sdk-go v0.3.5-0.20200824221853-d7b1fe2f9239/go.mod h1:/dKD5dZJ0bzDXrd/M3XMQmpXRk0bru3s9EMIeyBJfy4= -github.com/coinbase/rosetta-sdk-go v0.3.5-0.20200825180137-4b6b8c12e59e h1:gPomoE/+wkTz6/eFWdugcOvWlH/FjiKjNgsIrHrrhZ0= -github.com/coinbase/rosetta-sdk-go v0.3.5-0.20200825180137-4b6b8c12e59e/go.mod h1:ERvHFmf3+JUQB/6ZysASs8CCEcWJMAhN+y/ioqnBsLQ= -github.com/coinbase/rosetta-sdk-go v0.3.5-0.20200825192950-ca4a1cbb4439 h1:G6A5/O67JrbIKEJt/F3vbGWSiQjeUC0mHJFAnP40VtA= -github.com/coinbase/rosetta-sdk-go v0.3.5-0.20200825192950-ca4a1cbb4439/go.mod h1:ERvHFmf3+JUQB/6ZysASs8CCEcWJMAhN+y/ioqnBsLQ= -github.com/coinbase/rosetta-sdk-go v0.3.5-0.20200825193453-990ba41985fe h1:HKdKRwXIYzQqxf8hss365GMrYeKEBlVNe7aR8s15cfw= -github.com/coinbase/rosetta-sdk-go v0.3.5-0.20200825193453-990ba41985fe/go.mod h1:ERvHFmf3+JUQB/6ZysASs8CCEcWJMAhN+y/ioqnBsLQ= -github.com/coinbase/rosetta-sdk-go v0.3.5-0.20200825200012-1a2b012c535a h1:iVHj3zPYATksd7snj3zcuvuozEBpz6QmWXVxVgyAofY= -github.com/coinbase/rosetta-sdk-go v0.3.5-0.20200825200012-1a2b012c535a/go.mod h1:ERvHFmf3+JUQB/6ZysASs8CCEcWJMAhN+y/ioqnBsLQ= -github.com/coinbase/rosetta-sdk-go v0.3.5-0.20200825202822-ec2ccf5ca7b2 h1:czjlrSF+F+ZrZF38hPOqk7NcVCg9j7ITzbaWwNXcRYY= -github.com/coinbase/rosetta-sdk-go v0.3.5-0.20200825202822-ec2ccf5ca7b2/go.mod h1:ERvHFmf3+JUQB/6ZysASs8CCEcWJMAhN+y/ioqnBsLQ= -github.com/coinbase/rosetta-sdk-go v0.3.5-0.20200825203604-1e6ee29e1d18 h1:qGK7Gsn5TdVVyPGM3gTUhJQcVWKWcVTK7vOxMvqWs1I= -github.com/coinbase/rosetta-sdk-go v0.3.5-0.20200825203604-1e6ee29e1d18/go.mod h1:ERvHFmf3+JUQB/6ZysASs8CCEcWJMAhN+y/ioqnBsLQ= -github.com/coinbase/rosetta-sdk-go v0.3.5-0.20200825215532-31a92f0c1fb1 h1:Gc4zbvMijWKo9Wh8T1LnDDesViA2XK81UpA0vrfPisQ= -github.com/coinbase/rosetta-sdk-go v0.3.5-0.20200825215532-31a92f0c1fb1/go.mod h1:ERvHFmf3+JUQB/6ZysASs8CCEcWJMAhN+y/ioqnBsLQ= -github.com/coinbase/rosetta-sdk-go v0.3.5-0.20200825220011-d26cae449af1 h1:hBBzJARSy/TohObIr4SYnFWBYCDWg2rI5szOlmKO3CE= -github.com/coinbase/rosetta-sdk-go v0.3.5-0.20200825220011-d26cae449af1/go.mod h1:ERvHFmf3+JUQB/6ZysASs8CCEcWJMAhN+y/ioqnBsLQ= -github.com/coinbase/rosetta-sdk-go v0.3.5-0.20200825220704-401164ea449f h1:g30vKRyRFXzTTjM/+JldTv3oCyRN/fsqfK4G9itv4VY= -github.com/coinbase/rosetta-sdk-go v0.3.5-0.20200825220704-401164ea449f/go.mod h1:ERvHFmf3+JUQB/6ZysASs8CCEcWJMAhN+y/ioqnBsLQ= -github.com/coinbase/rosetta-sdk-go v0.3.5-0.20200825220938-e8c8639f9de9 h1:sp4HYBk/hBx84ANf9a6/hlHxkhko9GvgBDUkjyQH2xI= -github.com/coinbase/rosetta-sdk-go v0.3.5-0.20200825220938-e8c8639f9de9/go.mod h1:ERvHFmf3+JUQB/6ZysASs8CCEcWJMAhN+y/ioqnBsLQ= -github.com/coinbase/rosetta-sdk-go v0.3.5-0.20200825221259-01f4635468d6 h1:Sq7CNu6InK1Sa4mChRekU284Dw2FcSG8+Q3Ab8b/OAI= -github.com/coinbase/rosetta-sdk-go v0.3.5-0.20200825221259-01f4635468d6/go.mod h1:ERvHFmf3+JUQB/6ZysASs8CCEcWJMAhN+y/ioqnBsLQ= -github.com/coinbase/rosetta-sdk-go v0.3.5-0.20200825222405-5a0a89fadb36 h1:rct5jMSTTzxZuzTps1NxmnjoS4abr5fHch6Dql+yckw= -github.com/coinbase/rosetta-sdk-go v0.3.5-0.20200825222405-5a0a89fadb36/go.mod h1:ERvHFmf3+JUQB/6ZysASs8CCEcWJMAhN+y/ioqnBsLQ= -github.com/coinbase/rosetta-sdk-go v0.3.5-0.20200825232904-4639051d5690 h1:vVuAzyf8NtTE8uiUm7ypVdJlL+RLtUS2Tf9prMAQVFc= -github.com/coinbase/rosetta-sdk-go v0.3.5-0.20200825232904-4639051d5690/go.mod h1:ERvHFmf3+JUQB/6ZysASs8CCEcWJMAhN+y/ioqnBsLQ= -github.com/coinbase/rosetta-sdk-go v0.3.5-0.20200826005614-8eaafea66bf9 h1:PlhYUghjjT8RqyTtzfGdIoL4F7S50qK9m41RnpG2NlA= -github.com/coinbase/rosetta-sdk-go v0.3.5-0.20200826005614-8eaafea66bf9/go.mod h1:ERvHFmf3+JUQB/6ZysASs8CCEcWJMAhN+y/ioqnBsLQ= -github.com/coinbase/rosetta-sdk-go v0.3.5-0.20200826011303-e311283c1503 h1:rrBbzCG8CV1CzkGmSMufzadPiXIQg05jGRaglK/UKxY= -github.com/coinbase/rosetta-sdk-go v0.3.5-0.20200826011303-e311283c1503/go.mod h1:ERvHFmf3+JUQB/6ZysASs8CCEcWJMAhN+y/ioqnBsLQ= -github.com/coinbase/rosetta-sdk-go v0.3.5-0.20200826014430-cb55ba6945b5 h1:oSq8L9omheiEsNP/VBwrMsSfeggtvnsywL+c5QIcM6Y= -github.com/coinbase/rosetta-sdk-go v0.3.5-0.20200826014430-cb55ba6945b5/go.mod h1:ERvHFmf3+JUQB/6ZysASs8CCEcWJMAhN+y/ioqnBsLQ= -github.com/coinbase/rosetta-sdk-go v0.3.5-0.20200826042726-a9d02cfb0622 h1:3N+w0kT3GfVWFdvs0o+2DjP/WvX7C8RW3M7GtJuTf0Q= -github.com/coinbase/rosetta-sdk-go v0.3.5-0.20200826042726-a9d02cfb0622/go.mod h1:ERvHFmf3+JUQB/6ZysASs8CCEcWJMAhN+y/ioqnBsLQ= -github.com/coinbase/rosetta-sdk-go v0.3.5-0.20200826043835-9595c3a2c361 h1:puy1CPZtv3DPkNF8kCKaj6WdYYh4f2Pn9FLlCqKxcq0= -github.com/coinbase/rosetta-sdk-go v0.3.5-0.20200826043835-9595c3a2c361/go.mod h1:ERvHFmf3+JUQB/6ZysASs8CCEcWJMAhN+y/ioqnBsLQ= -github.com/coinbase/rosetta-sdk-go v0.3.5-0.20200826044804-4e9a062a840a h1:D5GhIG17ChlUYi5s4AECsp+o7GgFhEHQ6AZXBlMiDrk= -github.com/coinbase/rosetta-sdk-go v0.3.5-0.20200826044804-4e9a062a840a/go.mod h1:ERvHFmf3+JUQB/6ZysASs8CCEcWJMAhN+y/ioqnBsLQ= -github.com/coinbase/rosetta-sdk-go v0.3.5-0.20200826163301-f4e18c63d4d4 h1:9ragTKRH6HZitJ5zTA0dIoyP30kLQ/uyj65Ky6QYo4I= -github.com/coinbase/rosetta-sdk-go v0.3.5-0.20200826163301-f4e18c63d4d4/go.mod h1:ERvHFmf3+JUQB/6ZysASs8CCEcWJMAhN+y/ioqnBsLQ= -github.com/coinbase/rosetta-sdk-go v0.3.5-0.20200826173843-212f2d853cdc h1:VHW//kbg1wfvHz/1wOvPBiFh+ljbAYT+dXbZcuPjNAY= -github.com/coinbase/rosetta-sdk-go v0.3.5-0.20200826173843-212f2d853cdc/go.mod h1:ERvHFmf3+JUQB/6ZysASs8CCEcWJMAhN+y/ioqnBsLQ= -github.com/coinbase/rosetta-sdk-go v0.3.5-0.20200831181253-2c90cada22a0 h1:/IuP9Dle50p8d1/sN/A+zIMJ61j1hmc25lG9CzDb1Qw= -github.com/coinbase/rosetta-sdk-go v0.3.5-0.20200831181253-2c90cada22a0/go.mod h1:ERvHFmf3+JUQB/6ZysASs8CCEcWJMAhN+y/ioqnBsLQ= -github.com/coinbase/rosetta-sdk-go v0.3.5-0.20200831182036-449adc1bbece h1:BOSDN75he/HVJvCJzH2svBHXUTX+9v0X93bbRnduUeU= -github.com/coinbase/rosetta-sdk-go v0.3.5-0.20200831182036-449adc1bbece/go.mod h1:ERvHFmf3+JUQB/6ZysASs8CCEcWJMAhN+y/ioqnBsLQ= -github.com/coinbase/rosetta-sdk-go v0.3.5-0.20200831200955-3c1868507a81 h1:6R/X/QdYk8mdIrMo6bqC5RuaNbdmpJWRCWhrDl5db9Q= -github.com/coinbase/rosetta-sdk-go v0.3.5-0.20200831200955-3c1868507a81/go.mod h1:ERvHFmf3+JUQB/6ZysASs8CCEcWJMAhN+y/ioqnBsLQ= -github.com/coinbase/rosetta-sdk-go v0.3.5-0.20200901005342-2c16b7a9ff30 h1:beBdN7OZ0OO/wjADaievjIE0OAuk0qb4zBd2wNn9NfQ= -github.com/coinbase/rosetta-sdk-go v0.3.5-0.20200901005342-2c16b7a9ff30/go.mod h1:ERvHFmf3+JUQB/6ZysASs8CCEcWJMAhN+y/ioqnBsLQ= -github.com/coinbase/rosetta-sdk-go v0.3.5-0.20200901051936-be1d05c3bec0 h1:KIjma/2YyrC7S0UJoRO0Zdqneixkbks2uwe1lK5KfqU= -github.com/coinbase/rosetta-sdk-go v0.3.5-0.20200901051936-be1d05c3bec0/go.mod h1:ERvHFmf3+JUQB/6ZysASs8CCEcWJMAhN+y/ioqnBsLQ= -github.com/coinbase/rosetta-sdk-go v0.3.5-0.20200901204520-f16bdfd018fe h1:wl04MnWrSn1janJWwCvcQ8m6VaJop9BQHEd10f8ol0A= -github.com/coinbase/rosetta-sdk-go v0.3.5-0.20200901204520-f16bdfd018fe/go.mod h1:CE6c0Ws+rKwv4yiQDtAOuYwlC3tpXr2Cq5RTE/oVivY= -github.com/coinbase/rosetta-sdk-go v0.3.5-0.20200901205702-b739db12905a h1:virCY7xRX4Qi9EvWDordUyvCDlaP3k7ftIMLgcLzs1s= -github.com/coinbase/rosetta-sdk-go v0.3.5-0.20200901205702-b739db12905a/go.mod h1:CE6c0Ws+rKwv4yiQDtAOuYwlC3tpXr2Cq5RTE/oVivY= -github.com/coinbase/rosetta-sdk-go v0.4.1 h1:bL8qgwRZ6O56V+drjbdRAhnPZYdkNFjmY5V52v0AaxU= -github.com/coinbase/rosetta-sdk-go v0.4.1/go.mod h1:Luv0AhzZH81eul2hYZ3w0hBGwmFPiexwbntYxihEZck= -github.com/coinbase/rosetta-sdk-go v0.4.2-0.20200911162523-bbfbd83145d4 h1:aEQB1Y2qp036Cfy+Cm2pjtPEKmFofw/QKzfd3JL/wIU= -github.com/coinbase/rosetta-sdk-go v0.4.2-0.20200911162523-bbfbd83145d4/go.mod h1:Luv0AhzZH81eul2hYZ3w0hBGwmFPiexwbntYxihEZck= -github.com/coinbase/rosetta-sdk-go v0.4.2-0.20200912010944-f7d13f597014 h1:vMWAMtcGnpEhWh6kEDzDqviivHwFRp9w01MXGCLJCMU= -github.com/coinbase/rosetta-sdk-go v0.4.2-0.20200912010944-f7d13f597014/go.mod h1:Luv0AhzZH81eul2hYZ3w0hBGwmFPiexwbntYxihEZck= -github.com/coinbase/rosetta-sdk-go v0.4.2 h1:ojiNdDmmMwXIVIRN9vNCPytXtRBAm+EGCibJMhQ5z8o= -github.com/coinbase/rosetta-sdk-go v0.4.2/go.mod h1:Luv0AhzZH81eul2hYZ3w0hBGwmFPiexwbntYxihEZck= -github.com/coinbase/rosetta-sdk-go v0.4.3-0.20200914173330-69c73d3ff87a h1:lEI5Uy/qwhjzg+DCWJG94I+k3J+iG/zs6xjowmdzBvE= -github.com/coinbase/rosetta-sdk-go v0.4.3-0.20200914173330-69c73d3ff87a/go.mod h1:Luv0AhzZH81eul2hYZ3w0hBGwmFPiexwbntYxihEZck= -github.com/coinbase/rosetta-sdk-go v0.4.3-0.20200914180457-7d3458f14c0f h1:n6zV0BCXcVYilmKw8sNiRe5rc7K6R7Gb+vX25ZIvBsM= -github.com/coinbase/rosetta-sdk-go v0.4.3-0.20200914180457-7d3458f14c0f/go.mod h1:Luv0AhzZH81eul2hYZ3w0hBGwmFPiexwbntYxihEZck= -github.com/coinbase/rosetta-sdk-go v0.4.3 h1:qj1U6h7textY92kmjtQmWYN+REup17+cDcNVL9drPwA= -github.com/coinbase/rosetta-sdk-go v0.4.3/go.mod h1:Luv0AhzZH81eul2hYZ3w0hBGwmFPiexwbntYxihEZck= -github.com/coinbase/rosetta-sdk-go v0.4.4-0.20200916003715-bc5c0dc3f67f h1:Phzxs8VyctyVzRzMzAml/nLOYpNw/uTfD2cZe8wyEqU= -github.com/coinbase/rosetta-sdk-go v0.4.4-0.20200916003715-bc5c0dc3f67f/go.mod h1:Luv0AhzZH81eul2hYZ3w0hBGwmFPiexwbntYxihEZck= -github.com/coinbase/rosetta-sdk-go v0.4.4-0.20200916005026-5f676c313cfc h1:huzflMX5sJfrDIhv1p+D9U60jr/WC8qzJJ2EVgAazmU= -github.com/coinbase/rosetta-sdk-go v0.4.4-0.20200916005026-5f676c313cfc/go.mod h1:Luv0AhzZH81eul2hYZ3w0hBGwmFPiexwbntYxihEZck= -github.com/coinbase/rosetta-sdk-go v0.4.4 h1:zTUS4bVlTfD4xq/o6JtsuU+g9sf3+S3Nnn2A24Ycow4= -github.com/coinbase/rosetta-sdk-go v0.4.4/go.mod h1:Luv0AhzZH81eul2hYZ3w0hBGwmFPiexwbntYxihEZck= -github.com/coinbase/rosetta-sdk-go v0.4.5-0.20200918214028-4d968344e1a2 h1:SqJ4KAfZeyGiJc5GG9DD2vsOY/gsegJK9q9F8WaQRTc= -github.com/coinbase/rosetta-sdk-go v0.4.5-0.20200918214028-4d968344e1a2/go.mod h1:Luv0AhzZH81eul2hYZ3w0hBGwmFPiexwbntYxihEZck= -github.com/coinbase/rosetta-sdk-go v0.4.5-0.20200918214833-43eb6f18490b h1:BXy7lSdgwJeBRRo03QLzW4Bj5cRYGQJcjJKxRlUmicU= -github.com/coinbase/rosetta-sdk-go v0.4.5-0.20200918214833-43eb6f18490b/go.mod h1:Luv0AhzZH81eul2hYZ3w0hBGwmFPiexwbntYxihEZck= -github.com/coinbase/rosetta-sdk-go v0.4.5-0.20200918215107-9c608cf4c6da h1:4rH5i5ePaF5Q5IpX40ZO+0wUjvGY1WJoHfkYcbGcpYE= -github.com/coinbase/rosetta-sdk-go v0.4.5-0.20200918215107-9c608cf4c6da/go.mod h1:Luv0AhzZH81eul2hYZ3w0hBGwmFPiexwbntYxihEZck= -github.com/coinbase/rosetta-sdk-go v0.4.5-0.20200918215315-15f3222d97c4 h1:NNAwQM58a1eh/5vfrneTMeWG9hNrROlSj8Rwrc4TQW8= -github.com/coinbase/rosetta-sdk-go v0.4.5-0.20200918215315-15f3222d97c4/go.mod h1:Luv0AhzZH81eul2hYZ3w0hBGwmFPiexwbntYxihEZck= -github.com/coinbase/rosetta-sdk-go v0.4.5-0.20200918215605-d46fa023df73 h1:MDM59d52JqwUQFw0/Ow35IWxFQNkcTNIVO71MDRW9HM= -github.com/coinbase/rosetta-sdk-go v0.4.5-0.20200918215605-d46fa023df73/go.mod h1:Luv0AhzZH81eul2hYZ3w0hBGwmFPiexwbntYxihEZck= -github.com/coinbase/rosetta-sdk-go v0.4.5 h1:5Z+25mm/J1SStzSdTp5e8dlEMtCZRBZdOaTFCBfnP4Y= -github.com/coinbase/rosetta-sdk-go v0.4.5/go.mod h1:Luv0AhzZH81eul2hYZ3w0hBGwmFPiexwbntYxihEZck= -github.com/coinbase/rosetta-sdk-go v0.4.6 h1:zX2SLmBF1oFbK0c4QCMwkwcHN9VjenFfIt0Dr8k8JGY= -github.com/coinbase/rosetta-sdk-go v0.4.6/go.mod h1:Luv0AhzZH81eul2hYZ3w0hBGwmFPiexwbntYxihEZck= -github.com/coinbase/rosetta-sdk-go v0.4.7 h1:5KFc0CgLMkKamX++hYUFvE58a5/tCn0wSqpcTnDhRhY= -github.com/coinbase/rosetta-sdk-go v0.4.7/go.mod h1:8d4iN4VSGvLUzl+jRQlvYSLyS9TeY0QZebneWouizqU= -github.com/coinbase/rosetta-sdk-go v0.4.8 h1:+E1TM4q1c5/x/jE9FPI1IZIbNvUWy4tRRgDPLnKzUV4= -github.com/coinbase/rosetta-sdk-go v0.4.8/go.mod h1:8d4iN4VSGvLUzl+jRQlvYSLyS9TeY0QZebneWouizqU= -github.com/coinbase/rosetta-sdk-go v0.4.9 h1:Agy5+jrREo4Fa/0Jl0KEOvJDUxaImuCMD0qOxGiUwb4= -github.com/coinbase/rosetta-sdk-go v0.4.9/go.mod h1:8d4iN4VSGvLUzl+jRQlvYSLyS9TeY0QZebneWouizqU= -github.com/coinbase/rosetta-sdk-go v0.5.0 h1:WdHJJ7RSUJ1sXwxVAoUb/owH3GYVEK2K3GURIZT3NOA= -github.com/coinbase/rosetta-sdk-go v0.5.0/go.mod h1:QVVeKHWFNb0NyzEY06LxXMAylJkYa7n+Hk03pORr0ws= -github.com/coinbase/rosetta-sdk-go v0.5.1-0.20201006040540-21c3125fd72d h1:GOfiiYEBWClqpXwRR2o7tKDrXUN3GKWv8Qvmi+kblmU= -github.com/coinbase/rosetta-sdk-go v0.5.1-0.20201006040540-21c3125fd72d/go.mod h1:QVVeKHWFNb0NyzEY06LxXMAylJkYa7n+Hk03pORr0ws= -github.com/coinbase/rosetta-sdk-go v0.5.1-0.20201006163035-c3ad17e072a1 h1:feFtQluduAZKZBELfar7mm9XgkFTipct0q49iVww4Lw= -github.com/coinbase/rosetta-sdk-go v0.5.1-0.20201006163035-c3ad17e072a1/go.mod h1:QVVeKHWFNb0NyzEY06LxXMAylJkYa7n+Hk03pORr0ws= -github.com/coinbase/rosetta-sdk-go v0.5.1 h1:UGuEt/nMWHvnLmWZqJdoQP+HhWL5/6K04rjhPHhmS2M= -github.com/coinbase/rosetta-sdk-go v0.5.1/go.mod h1:QVVeKHWFNb0NyzEY06LxXMAylJkYa7n+Hk03pORr0ws= -github.com/coinbase/rosetta-sdk-go v0.5.2-0.20201006190307-bf4606611446 h1:KbefWZdPOb0yBl5DGV/FfJ95OJMiHxdxiJGLY+EjlT0= -github.com/coinbase/rosetta-sdk-go v0.5.2-0.20201006190307-bf4606611446/go.mod h1:QVVeKHWFNb0NyzEY06LxXMAylJkYa7n+Hk03pORr0ws= -github.com/coinbase/rosetta-sdk-go v0.5.2 h1:G91QWW0wrqmMhW2vJ4LP9P3gVukZs7df2Gf6FF1uff0= -github.com/coinbase/rosetta-sdk-go v0.5.2/go.mod h1:QVVeKHWFNb0NyzEY06LxXMAylJkYa7n+Hk03pORr0ws= github.com/coinbase/rosetta-sdk-go v0.5.3-0.20201015002757-978c228143a3 h1:p8p0dVZDh/Zs35tzQq3nFftgFbphNvSzZ6dzLT31F7I= github.com/coinbase/rosetta-sdk-go v0.5.3-0.20201015002757-978c228143a3/go.mod h1:QVVeKHWFNb0NyzEY06LxXMAylJkYa7n+Hk03pORr0ws= github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= +github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk= github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= +github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE= @@ -197,13 +105,8 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/deckarep/golang-set v0.0.0-20180603214616-504e848d77ea/go.mod h1:93vsz/8Wt4joVM7c2AVqh+YRMiUSc14yDtF28KmMOgQ= github.com/decred/dcrd/lru v1.0.0/go.mod h1:mxKOwFd7lFjN2GZYsiz/ecgqR6kkYAl+0pz0tEMk218= -github.com/dgraph-io/badger/v2 v2.0.3 h1:inzdf6VF/NZ+tJ8RwwYMjJMvsOALTHYdozn0qSl6XJI= -github.com/dgraph-io/badger/v2 v2.0.3/go.mod h1:3KY8+bsP8wI0OEnQJAKpd4wIJW/Mm32yw2j/9FUVnIM= -github.com/dgraph-io/badger/v2 v2.2007.1 h1:t36VcBCpo4SsmAD5M8wVv1ieVzcALyGfaJ92z4ccULM= -github.com/dgraph-io/badger/v2 v2.2007.1/go.mod h1:26P/7fbL4kUZVEVKLAKXkBXKOydDmM2p1e+NhhnBCAE= github.com/dgraph-io/badger/v2 v2.2007.2 h1:EjjK0KqwaFMlPin1ajhP943VPENHJdEz1KLIegjaI3k= github.com/dgraph-io/badger/v2 v2.2007.2/go.mod h1:26P/7fbL4kUZVEVKLAKXkBXKOydDmM2p1e+NhhnBCAE= -github.com/dgraph-io/ristretto v0.0.2-0.20200115201040-8f368f2f2ab3/go.mod h1:KPxhHT9ZxKefz+PCeOGsrHpl1qZ7i70dGTu2u+Ahh6E= github.com/dgraph-io/ristretto v0.0.3-0.20200630154024-f66de99634de/go.mod h1:KPxhHT9ZxKefz+PCeOGsrHpl1qZ7i70dGTu2u+Ahh6E= github.com/dgraph-io/ristretto v0.0.3 h1:jh22xisGBjrEVnRZ1DVTpBVQm0Xndu8sMl0CWDzSIBI= github.com/dgraph-io/ristretto v0.0.3/go.mod h1:KPxhHT9ZxKefz+PCeOGsrHpl1qZ7i70dGTu2u+Ahh6E= @@ -215,7 +118,6 @@ github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13/go.mod h1:SqUrOPUn github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= github.com/dlclark/regexp2 v1.2.0/go.mod h1:2pZnwuY/m+8K6iRw6wQdMtk+rH5tNGR1i55kozfMjCc= github.com/docker/docker v1.4.2-0.20180625184442-8e610b2b55bf/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= -github.com/dop251/goja v0.0.0-20200219165308-d1232e640a87/go.mod h1:Mw6PkjjMXWbTj+nnj4s3QPXq1jaT0s5pC0iFD4+BOAA= github.com/dop251/goja v0.0.0-20200721192441-a695b0cdd498/go.mod h1:Mw6PkjjMXWbTj+nnj4s3QPXq1jaT0s5pC0iFD4+BOAA= github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo= github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= @@ -223,15 +125,10 @@ github.com/dvyukov/go-fuzz v0.0.0-20200318091601-be3528f3a813/go.mod h1:11Gm+ccJ github.com/edsrzf/mmap-go v0.0.0-20160512033002-935e0e8a636c/go.mod h1:YO35OhQPt3KJa3ryjFM5Bs14WD66h8eGKpfaBNrHW5M= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= -github.com/ethereum/go-ethereum v1.9.18 h1:+vzvufVD7+OfQa07IJP20Z7AGZsJaw0M6JIA/WQcqy8= -github.com/ethereum/go-ethereum v1.9.18/go.mod h1:JSSTypSMTkGZtAdAChH2wP5dZEvPGh3nUTuDpH+hNrg= -github.com/ethereum/go-ethereum v1.9.20 h1:kk/J5OIoaoz3DRrCXznz3RGi212mHHXwzXlY/ZQxcj0= -github.com/ethereum/go-ethereum v1.9.20/go.mod h1:JSSTypSMTkGZtAdAChH2wP5dZEvPGh3nUTuDpH+hNrg= -github.com/ethereum/go-ethereum v1.9.21 h1:8qRlhzrItnmUGdVlBzZLI2Tb46S0RdSNjFwICo781ws= -github.com/ethereum/go-ethereum v1.9.21/go.mod h1:RXAVzbGrSGmDkDnHymruTAIEjUR3E4TX0EOpaj702sI= github.com/ethereum/go-ethereum v1.9.22 h1:/Fea9n2EWJuNJ9oahMq9luqjRBcbW7QWdThbcJl13ek= github.com/ethereum/go-ethereum v1.9.22/go.mod h1:FQjK3ZwD8C5DYn7ukTmFee36rq1dOMESiUfXr5RUc1w= github.com/fatih/color v1.3.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= +github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/fatih/color v1.9.0 h1:8xPHl4/q1VyqGIPif1F+1V3Y3lSmrq01EabUW3CoW5s= github.com/fatih/color v1.9.0/go.mod h1:eQcE1qtQxscV5RaZvpXrrb8Drkc3/DdQ+uUYCNjL+zU= github.com/fatih/structtag v1.2.0 h1:/OdNE99OxoI/PqaW/SuSK9uxxT3f/tcSZgon/ssNSx4= @@ -241,6 +138,7 @@ github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMo github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= github.com/gballet/go-libpcsclite v0.0.0-20190607065134-2772fd86a8ff/go.mod h1:x7DCsMOv1taUwEWCzT4cmDeAkigA5/QCwUodaVOe8Ww= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= +github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= @@ -254,10 +152,11 @@ github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zV github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1 h1:YF8+flBXS5eO826T4nzqPrxfhQThhXl0YzfuUPu4SBg= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.3.2-0.20190517061210-b285ee9cfc6c/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.4/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= @@ -271,10 +170,9 @@ github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.2-0.20200707131729-196ae77b8a26 h1:lMm2hD9Fy0ynom5+85/pbdkiYcBqM1JWmhpAXLmy0fw= github.com/golang/snappy v0.0.2-0.20200707131729-196ae77b8a26/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= -github.com/google/addlicense v0.0.0-20200817051935-6f4cd4aacc89 h1:oTppmscIAQ2Y1tcsMDcTLR3z4MN/96/pvIsBSLGl7o8= -github.com/google/addlicense v0.0.0-20200817051935-6f4cd4aacc89/go.mod h1:EMjYTRimagHs1FwlIqKyX3wAM0u3rA+McvlIIWmSamA= github.com/google/addlicense v0.0.0-20200827091314-d1655b921368 h1:Ds6gDZHoviaQM7r7oMx/cG2qwZc3l5u7cg6gTkxOZNE= github.com/google/addlicense v0.0.0-20200827091314-d1655b921368/go.mod h1:EMjYTRimagHs1FwlIqKyX3wAM0u3rA+McvlIIWmSamA= +github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= @@ -284,20 +182,45 @@ github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.0 h1:/QaMHBdZ26BB3SSst0Iwl10Epc+xhTquomWX0oZEB6w= github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.2 h1:X2ev0eStA3AbceY54o37/0PQ/UWqKEiiO2dKL5OPaFM= github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/gofuzz v1.1.1-0.20200604201612-c04b05f3adfa/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= github.com/google/pprof v0.0.0-20181127221834-b4f47329b966/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= +github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= +github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= -github.com/gorilla/mux v1.7.4/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= +github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= +github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= +github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= -github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= github.com/gorilla/websocket v1.4.1-0.20190629185528-ae1634f6a989/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= +github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/graph-gophers/graphql-go v0.0.0-20191115155744-f33e81362277/go.mod h1:9CQHMSxwO4MprSdzoIEobiHpoLtHm77vfxsvsIN5Vuc= github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= +github.com/hashicorp/consul/api v1.1.0/go.mod h1:VmuI/Lkw1nC05EYQWNKwWGbkg+FbDBtguAZLlVdkD9Q= +github.com/hashicorp/consul/sdk v0.1.1/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8= +github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= +github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= +github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= +github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM= +github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk= +github.com/hashicorp/go-rootcerts v1.0.0/go.mod h1:K6zTfqpRlCUIjkwsN4Z+hiSfzSTQa6eBIzfwKfwNnHU= +github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU= +github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4= +github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= +github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= +github.com/hashicorp/go.net v0.0.1/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA0oac0k90= +github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= +github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64= +github.com/hashicorp/mdns v1.0.0/go.mod h1:tL+uN++7HEJ6SQLQ2/p+z2pH24WQKWjBPkE0mNTz8vQ= +github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I= +github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc= github.com/holiman/uint256 v1.1.1/go.mod h1:y4ga/t+u+Xwd7CpDgZESaRcWy0I7XMlTMA25ApIH5Jw= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/huin/goupnp v1.0.0/go.mod h1:n9v9KO1tAxYH82qOn+UTIFQDmx5n1Zxd/ClZDMX7Bnc= @@ -309,11 +232,12 @@ github.com/influxdata/influxdb v1.2.3-0.20180221223340-01288bdb0883/go.mod h1:qZ github.com/jackpal/go-nat-pmp v1.0.2-0.20160603034137-1fa385a6f458/go.mod h1:QPH045xvCAeXUZOxsnwmrtiCoxIr9eob+4orBN1SBKc= github.com/jessevdk/go-flags v0.0.0-20141203071132-1679536dcc89/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= -github.com/jinzhu/copier v0.0.0-20190924061706-b57f9002281a h1:zPPuIq2jAWWPTrGt70eK/BSch+gFAGrNzecsoENgu2o= -github.com/jinzhu/copier v0.0.0-20190924061706-b57f9002281a/go.mod h1:yL958EeXv8Ylng6IfnvG4oflryUi3vgA3xPs9hmII1s= github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= github.com/jrick/logrotate v1.0.0/go.mod h1:LNinyqDIJnpAur+b8yyulnQw/wDuN1+BYKlTRt3OuAQ= +github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= +github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= +github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/julienschmidt/httprouter v1.1.1-0.20170430222011-975b5c4c7c21/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/karalabe/usb v0.0.0-20190919080040-51dc0efba356/go.mod h1:Od972xHfMJowv7NGVDiWVxk2zxnWgjLlJzE+F4F7AGU= @@ -333,6 +257,8 @@ github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+ github.com/lucasjones/reggen v0.0.0-20180717132126-cdb49ff09d77 h1:6xiz3+ZczT3M4+I+JLpcPGG1bQKm8067HktB17EDWEE= github.com/lucasjones/reggen v0.0.0-20180717132126-cdb49ff09d77/go.mod h1:5ELEyG+X8f+meRWHuqUOewBOhvHkl7M76pdGEansxW4= github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= +github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= +github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= github.com/mattn/go-colorable v0.1.0/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= github.com/mattn/go-colorable v0.1.4 h1:snbPLB8fVfU9iwbbo30TPtbLRzwWu6aJS6Xh4eaaviA= github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= @@ -340,6 +266,7 @@ github.com/mattn/go-colorable v0.1.7 h1:bQGKb3vps/j0E9GfJQ03JyhRuxsvdAanXlT9BTw3 github.com/mattn/go-colorable v0.1.7/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= github.com/mattn/go-ieproxy v0.0.0-20190610004146-91bb50d98149/go.mod h1:31jz6HNzdxOmlERGGEc4v/dMssOfmp2p5bT/okiKFFc= github.com/mattn/go-ieproxy v0.0.0-20190702010315-6dee0af9227d/go.mod h1:31jz6HNzdxOmlERGGEc4v/dMssOfmp2p5bT/okiKFFc= +github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-isatty v0.0.5-0.20180830101745-3fb116b82035/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.10/go.mod h1:qgIWMr58cqv1PHHyhnkY9lrL7etaEgOFcMEpPG5Rm84= @@ -353,10 +280,19 @@ github.com/mattn/go-runewidth v0.0.4/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzp github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b h1:j7+1HpAFS1zy5+Q4qx1fWh90gTKwiN4QCGoY9TWyyO4= github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b/go.mod h1:01TrycV0kFyexm33Z7vhZRXopbI8J3TDReVlkTgMUxE= +github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= +github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc= +github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= +github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI= +github.com/mitchellh/gox v0.4.0/go.mod h1:Sd9lOJ0+aimLBi73mGofS1ycjY8lL3uZM3JPS42BGNg= +github.com/mitchellh/iochan v1.0.0/go.mod h1:JwYml1nuB7xOzsp52dPpHFffvOCDupsG0QubkSMEySY= +github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/mitchellh/mapstructure v1.3.3 h1:SzB1nHZ2Xi+17FP0zVQBHIZqvwRN9408fJO8h+eeNA8= github.com/mitchellh/mapstructure v1.3.3/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/naoina/go-stringutil v0.1.0/go.mod h1:XJ2SJL9jCtBh+P9q5btrd/Ylo8XwT/h1USek5+NqSA0= github.com/naoina/toml v0.1.2-0.20170918210437-9fafd6967416/go.mod h1:NBIhNtsFMo3G2szEBne+bO4gS192HuIYRqfvOWb4i1E= @@ -376,6 +312,7 @@ github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1Cpa github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= +github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= github.com/pborman/uuid v0.0.0-20170112150404-1b00554d8222/go.mod h1:VyrYX9gd7irzKovcSS6BIIEwPRkP2Wm2m9ufcdFSJ34= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= github.com/peterh/liner v1.1.1-0.20190123174540-a2c9a5303de7/go.mod h1:CRroGNssyjTd/qIG2FyxByd2S8JEAZXBl4qUrZf8GS0= @@ -386,6 +323,7 @@ github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= @@ -407,7 +345,8 @@ github.com/rs/cors v0.0.0-20160617231935-a62a804a8a00/go.mod h1:gFx+x8UowdsKA9Ac github.com/rs/xhandler v0.0.0-20160618193221-ed27b6fd6521/go.mod h1:RvLn4FgxWubrpZHtQLnOf6EwhN2hEMusxZOhcW9H3UQ= github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= -github.com/segmentio/golines v0.0.0-20200808004416-0a9796b248e8/go.mod h1:bQSh5qdVR67XiCKbaVvYO41s50c5hQo+3cY/1CQQ3xQ= +github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= +github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= github.com/segmentio/golines v0.0.0-20200824192126-7f30d3046793 h1:rhR7esJSmty+9ST6Gsp7mlQHkpISw2DiYjuFaz3dRDg= github.com/segmentio/golines v0.0.0-20200824192126-7f30d3046793/go.mod h1:bQSh5qdVR67XiCKbaVvYO41s50c5hQo+3cY/1CQQ3xQ= github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= @@ -418,6 +357,8 @@ github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeV github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.2 h1:SPIRibHv4MatM3XXNO2BJeFLZwZ2LvZgfQ5+UNI2im4= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= +github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= +github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI= @@ -426,15 +367,15 @@ github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= github.com/spf13/cobra v0.0.5 h1:f0B+LkLX6DtmRH1isoNA9VTtNUK9K8xYd28JNNfOv/s= github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU= -github.com/spf13/cobra v1.0.0 h1:6m/oheQuQ13N9ks4hubMG6BnvwOeaJrqSPLahSnczz8= -github.com/spf13/cobra v1.0.0/go.mod h1:/6GTrnGXV9HjY+aR4k0oJ5tcvakLuG6EuKReYlHNrgE= +github.com/spf13/cobra v1.1.0 h1:aq3wCKjTPmzcNWLVGnsFVN4rflK7Uzn10F8/aw8MhdQ= +github.com/spf13/cobra v1.1.0/go.mod h1:yk5b0mALVusDL5fMM6Rd1wgnoO5jUPhwsQ6LQAJTidQ= github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= github.com/spf13/pflag v1.0.3 h1:zPAT6CGy6wXeQ7NtTnaTerfKOsV6V6F8agHXFiazDkg= github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s= -github.com/spf13/viper v1.4.0/go.mod h1:PTJ7Z/lr49W6bUbkmS1V3by4uWynFiR9p7+dSq/yZzE= +github.com/spf13/viper v1.7.0/go.mod h1:8WkrPz2fc9jxqZNCJI/76HCieCp4Q8HaLFoCha5qpdg= github.com/status-im/keycard-go v0.0.0-20190316090335-8537d3370df4/go.mod h1:RZLeN1LMWmRsyYjvAu+I6Dm9QmlDaIIt+Y+4Kd7Tp+Q= github.com/steakknife/bloomfilter v0.0.0-20180922174646-6819c0d2a570 h1:gIlAHnH1vJb5vwEjIp5kBj/eu99p/bl0Ay2goiPe5xE= github.com/steakknife/bloomfilter v0.0.0-20180922174646-6819c0d2a570/go.mod h1:8OR4w3TdeIHIh1g6EMY5p0gVNOovcWC+1vpc7naMuAw= @@ -451,26 +392,18 @@ github.com/stretchr/testify v1.5.1 h1:nOGnQDM7FYENwehXlg/kFVnos3rEvtKTjRvOWSzb6H github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/syndtr/goleveldb v1.0.1-0.20190923125748-758128399b1d/go.mod h1:9OrXJhf154huy1nPWmuSrkgjPUtUNhA+Zmy+6AESzuA= +github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= github.com/syndtr/goleveldb v1.0.1-0.20200815110645-5c35d600f0ca/go.mod h1:u2MKkTVTVJWe5D1rCvame8WqhBd88EuIwODJZ1VHCPM= -github.com/tidwall/gjson v1.6.0 h1:9VEQWz6LLMUsUl6PueE49ir4Ka6CzLymOAZDxpFsTDc= -github.com/tidwall/gjson v1.6.0/go.mod h1:P256ACg0Mn+j1RXIDXoss50DeIABTYK1PULOJHhxOls= github.com/tidwall/gjson v1.6.1 h1:LRbvNuNuvAiISWg6gxLEFuCe72UKy5hDqhxW/8183ws= github.com/tidwall/gjson v1.6.1/go.mod h1:BaHyNc5bjzYkPqgLq7mdVzeiRtULKULXLgZFKsxEHI0= github.com/tidwall/match v1.0.1 h1:PnKP62LPNxHKTwvHHZZzdOAOCtsJTjo6dZLCwpKm5xc= github.com/tidwall/match v1.0.1/go.mod h1:LujAq0jyVjBy028G1WhWfIzbpQfMO8bBZ6Tyb0+pL9E= -github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= -github.com/tidwall/pretty v1.0.1 h1:WE4RBSZ1x6McVVC8S/Md+Qse8YUv6HRObAx6ke00NY8= -github.com/tidwall/pretty v1.0.1/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= github.com/tidwall/pretty v1.0.2 h1:Z7S3cePv9Jwm1KwS0513MRaoUe3S01WPbLNV40pwWZU= github.com/tidwall/pretty v1.0.2/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= -github.com/tidwall/sjson v1.1.1 h1:7h1vk049Jnd5EH9NyzNiEuwYW4b5qgreBbqRC19AS3U= -github.com/tidwall/sjson v1.1.1/go.mod h1:yvVuSnpEQv5cYIrO+AT6kw4QVfd5SDZoGIS7/5+fZFs= github.com/tidwall/sjson v1.1.2 h1:NC5okI+tQ8OG/oyzchvwXXxRxCV/FVdhODbPKkQ25jQ= github.com/tidwall/sjson v1.1.2/go.mod h1:SEzaDwxiPzKzNfUEO4HbYF/m4UCSJDsGgNqsS1LvdoY= github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/tyler-smith/go-bip39 v1.0.1-0.20181017060643-dbb3b84ba2ef/go.mod h1:sJ5fKU0s6JVwZjjcUEX2zFOnvq0ASQ2K9Zr6cf67kNs= -github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc= github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= github.com/vmihailenco/msgpack/v4 v4.3.11/go.mod h1:gborTTJjAo/GWTqqRjrLCn9pgNN+NXzzngzBKDPIqw4= @@ -487,35 +420,46 @@ github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= +go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= +go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= golang.org/x/arch v0.0.0-20180920145803-b19384d3c130/go.mod h1:cYlCBUl1MsqxdiKgmc4uh7TxZfWSFLOGSRR090WDxt8= golang.org/x/crypto v0.0.0-20170930174604-9419663f5a44/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20181127143415-eb0de9b17e85/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550 h1:ObdrDkeb4kJdCP557AjRjq69pTHfNouLtWZG7j9rPN8= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200115085410-6d4e4cb37c7d/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200510223506-06a226fb4e37/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 h1:psW17arqaxU48Z5kZ0CQnkZWQJsqcURM6tKiBApRjXI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20200728195943-123391ffb6de h1:ikNHVSjEfnvz6sxdSPCaPt572qowuyMDMJLLm3Db3ig= -golang.org/x/crypto v0.0.0-20200728195943-123391ffb6de/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200820211705-5c72a883971a h1:vclmkQCjlDX5OydZ9wv8rBCcS0QyQY66Mpf/7BZbInM= golang.org/x/crypto v0.0.0-20200820211705-5c72a883971a/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= golang.org/x/exp v0.0.0-20190731235908-ec7cb31e5a56/go.mod h1:JhuoJpWY28nO4Vef9tZUw9qufEGTyX1+7lmHxV5q5G4= +golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek= +golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= +golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3 h1:XQyxROzUlZH+WIQwySDgnISgOivlhjIEwaQaJEJrrN0= golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= +golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= golang.org/x/mobile v0.0.0-20200801112145-973feb4309de/go.mod h1:skQtrUTUwhdJvXM/2KKJzY8pDgNr9I/FOMqDVRPBUS4= golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= @@ -529,12 +473,16 @@ golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73r golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181011144130-49bb7cea24b1/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181023162649-9b4f9f5ad519/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190522155817-f3200d17e092/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= +golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= golang.org/x/net v0.0.0-20190620200207-3b0461eec859 h1:R/3boaszxrf1GEUWTVDzSKVwLmSJpwZ1yqXm8j0v2QI= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= @@ -549,24 +497,34 @@ golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81R golang.org/x/net v0.0.0-20200904194848-62affa334b73 h1:MXfv8rhZWmFeqX3GNZRsd6vOLoaCHjYEX3qkRo3YBUA= golang.org/x/net v0.0.0-20200904194848-62affa334b73/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= +golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208 h1:qwRHBd0NqMbJxfbotnDhm2ByMI1Shq4Y6oRJo21SGJA= golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180903190138-2b024373dcd9/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181026203630-95b1ffbd15a5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb h1:fgwFCsaw9buMuxNd6+DQfAuSFqbNiQZpcgJQAgJsK6k= golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -581,14 +539,14 @@ golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200519105757-fe76b779f299/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200814200057-3d37ad5750ed/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200817155316-9781c653f443 h1:X18bCaipMcoJGm27Nv7zr4XYPKGUy92GtqboKC2Hxaw= -golang.org/x/sys v0.0.0-20200817155316-9781c653f443/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200824131525-c12d262b63d8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200909081042-eff7692f9009 h1:W0lCpv29Hv0UaM1LXb9QlBHLNP8UFfcKjblhVCWftOM= golang.org/x/sys v0.0.0-20200909081042-eff7692f9009/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -597,9 +555,20 @@ golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGm golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191024220359-3d91e92cde03 h1:4gtJXHJ9ud0q8MNSDxJsRU/WH+afypbe4Vk4zq+8qow= golang.org/x/tools v0.0.0-20191024220359-3d91e92cde03/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191112195655-aa38f8e97acc/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191130070609-6e064ea0cf2d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20200117012304-6edc0a871e69/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= @@ -615,15 +584,30 @@ golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IV golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= +google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= +google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= +google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= +google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= google.golang.org/appengine v1.6.5 h1:tycE03LOZYQNhDpS27tcQdAzLCVMaj7QT2SXxebnpCM= google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= +google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8= +google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= -google.golang.org/grpc v1.21.0/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= +google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= +google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= @@ -647,6 +631,7 @@ gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogR gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= +gopkg.in/ini.v1 v1.51.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/natefinch/npipe.v2 v2.0.0-20160621034901-c1b8fa8bdcce/go.mod h1:5AcXVHNjg+BDxry382+8OKon8SEWiKktQR07RKPsv1c= gopkg.in/olebedev/go-duktape.v3 v3.0.0-20200619000410-60c24ae608a6/go.mod h1:uAJfkITjFhyEEuUfm7bsmCZRbW5WRq8s9EY8HZ6hCns= gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= @@ -663,11 +648,13 @@ gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= honnef.co/go/tools v0.0.1-2020.1.5/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= mvdan.cc/gofumpt v0.0.0-20200709182408-4fd085cb6d5f/go.mod h1:9VQ397fNXEnF84t90W4r4TRCQK+pg9f8ugVfyj+S26w= mvdan.cc/gofumpt v0.0.0-20200802201014-ab5a8192947d/go.mod h1:bzrjFmaD6+xqohD3KYP0H2FEuxknnBmyyOxdhLdaIws= mvdan.cc/xurls/v2 v2.2.0/go.mod h1:EV1RMtya9D6G5DMYPGD8zTQzaHet6Jh8gFlRgGRJeO8= -src.techknowlogick.com/xgo v1.1.1-0.20200814033943-12cf2e8194ca h1:kumZMjGGyzQDXLgV1CywLHdhnLtaFxDui/2IXqPtVMw= -src.techknowlogick.com/xgo v1.1.1-0.20200814033943-12cf2e8194ca/go.mod h1:31CE1YKtDOrKTk9PSnjTpe6YbO6W/0LTYZ1VskL09oU= +rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= From 5abf73cb37acc0783beb01cb31de14c03f22cba5 Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Thu, 15 Oct 2020 08:51:49 -0700 Subject: [PATCH 110/376] Log reconciler queue size info --- pkg/logger/logger.go | 4 +++- pkg/results/data_results.go | 28 ++++++++++++++++++---------- pkg/tester/data.go | 10 +++++++++- 3 files changed, 30 insertions(+), 12 deletions(-) diff --git a/pkg/logger/logger.go b/pkg/logger/logger.go index 5074fe2c..04463b1f 100644 --- a/pkg/logger/logger.go +++ b/pkg/logger/logger.go @@ -122,12 +122,14 @@ func (l *Logger) LogDataStatus(ctx context.Context, status *results.CheckDataSta } progressMessage := fmt.Sprintf( - "[PROGRESS] Blocks Synced: %d/%d (Completed: %f%%, Rate: %f/second) Time Remaining: %s", + "[PROGRESS] Blocks Synced: %d/%d (Completed: %f%%, Rate: %f/second) Time Remaining: %s Reconciler Queue: %d (Last Index Checked: %d)", status.Progress.Blocks, status.Progress.Tip, status.Progress.Completed, status.Progress.Rate, status.Progress.TimeRemaining, + status.Progress.ReconcilerQueueSize, + status.Progress.ReconcilerLastIndex, ) // Don't print out the same progress message twice. diff --git a/pkg/results/data_results.go b/pkg/results/data_results.go index a39a5061..99f21045 100644 --- a/pkg/results/data_results.go +++ b/pkg/results/data_results.go @@ -27,6 +27,7 @@ import ( "github.com/coinbase/rosetta-sdk-go/asserter" "github.com/coinbase/rosetta-sdk-go/fetcher" + "github.com/coinbase/rosetta-sdk-go/reconciler" "github.com/coinbase/rosetta-sdk-go/storage" "github.com/coinbase/rosetta-sdk-go/syncer" "github.com/coinbase/rosetta-sdk-go/types" @@ -228,11 +229,13 @@ func ComputeCheckDataStats( // CheckDataProgress contains information // about check:data's syncing progress. type CheckDataProgress struct { - Blocks int64 `json:"blocks"` - Tip int64 `json:"tip"` - Completed float64 `json:"completed"` - Rate float64 `json:"rate"` - TimeRemaining string `json:"time_remaining"` + Blocks int64 `json:"blocks"` + Tip int64 `json:"tip"` + Completed float64 `json:"completed"` + Rate float64 `json:"rate"` + TimeRemaining string `json:"time_remaining"` + ReconcilerQueueSize int `json:"reconciler_queue_size"` + ReconcilerLastIndex int64 `json:"reconciler_last_index"` } // ComputeCheckDataProgress returns @@ -242,6 +245,7 @@ func ComputeCheckDataProgress( fetcher *fetcher.Fetcher, network *types.NetworkIdentifier, counters *storage.CounterStorage, + reconciler *reconciler.Reconciler, ) *CheckDataProgress { networkStatus, fetchErr := fetcher.NetworkStatusRetry(ctx, network, nil) if fetchErr != nil { @@ -287,11 +291,13 @@ func ComputeCheckDataProgress( blocksSyncedFloat, _ := blocksSynced.Float64() return &CheckDataProgress{ - Blocks: adjustedBlocks, - Tip: tipIndex, - Completed: blocksSyncedFloat * utils.OneHundred, - Rate: blocksPerSecondFloat, - TimeRemaining: utils.TimeToTip(blocksPerSecondFloat, adjustedBlocks, tipIndex).String(), + Blocks: adjustedBlocks, + Tip: tipIndex, + Completed: blocksSyncedFloat * utils.OneHundred, + Rate: blocksPerSecondFloat, + TimeRemaining: utils.TimeToTip(blocksPerSecondFloat, adjustedBlocks, tipIndex).String(), + ReconcilerQueueSize: reconciler.QueueSize(), + ReconcilerLastIndex: reconciler.LastIndexReconciled(), } } @@ -310,6 +316,7 @@ func ComputeCheckDataStatus( balances *storage.BalanceStorage, fetcher *fetcher.Fetcher, network *types.NetworkIdentifier, + reconciler *reconciler.Reconciler, ) *CheckDataStatus { return &CheckDataStatus{ Stats: ComputeCheckDataStats( @@ -322,6 +329,7 @@ func ComputeCheckDataStatus( fetcher, network, counters, + reconciler, ), } } diff --git a/pkg/tester/data.go b/pkg/tester/data.go index d8b8e9bc..dbbf05a0 100644 --- a/pkg/tester/data.go +++ b/pkg/tester/data.go @@ -361,7 +361,14 @@ func (t *DataTester) StartPeriodicLogger( big.NewInt(periodicLoggingSeconds), ) - status := results.ComputeCheckDataStatus(ctx, t.counterStorage, t.balanceStorage, t.fetcher, t.config.Network) + status := results.ComputeCheckDataStatus( + ctx, + t.counterStorage, + t.balanceStorage, + t.fetcher, + t.config.Network, + t.reconciler, + ) t.logger.LogDataStatus(ctx, status) } } @@ -378,6 +385,7 @@ func (t *DataTester) ServeHTTP(w http.ResponseWriter, r *http.Request) { t.balanceStorage, t.fetcher, t.network, + t.reconciler, ) if err := json.NewEncoder(w).Encode(status); err != nil { From 29bfe144d32dccb758a01da489a6dd25659bd441 Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Thu, 15 Oct 2020 08:52:52 -0700 Subject: [PATCH 111/376] Update version --- cmd/root.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/root.go b/cmd/root.go index 4b9687e3..d276a4dd 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -247,6 +247,6 @@ var versionCmd = &cobra.Command{ Use: "version", Short: "Print rosetta-cli version", Run: func(cmd *cobra.Command, args []string) { - fmt.Println("v0.5.12") + fmt.Println("v0.5.13") }, } From d00637f0d98b328c605cd99e65fc21d542ea61bd Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Thu, 15 Oct 2020 09:02:33 -0700 Subject: [PATCH 112/376] Fix linting issue --- pkg/logger/logger.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/logger/logger.go b/pkg/logger/logger.go index 04463b1f..c639a2a6 100644 --- a/pkg/logger/logger.go +++ b/pkg/logger/logger.go @@ -122,7 +122,7 @@ func (l *Logger) LogDataStatus(ctx context.Context, status *results.CheckDataSta } progressMessage := fmt.Sprintf( - "[PROGRESS] Blocks Synced: %d/%d (Completed: %f%%, Rate: %f/second) Time Remaining: %s Reconciler Queue: %d (Last Index Checked: %d)", + "[PROGRESS] Blocks Synced: %d/%d (Completed: %f%%, Rate: %f/second) Time Remaining: %s Reconciler Queue: %d (Last Index Checked: %d)", // nolint:lll status.Progress.Blocks, status.Progress.Tip, status.Progress.Completed, From f84cbf860f51d6cf89542ace8362d3d3c32c97f3 Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Thu, 15 Oct 2020 09:10:10 -0700 Subject: [PATCH 113/376] Update rosetta-sdk-go --- go.mod | 2 +- go.sum | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index 59790fac..be91211f 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/coinbase/rosetta-cli go 1.13 require ( - github.com/coinbase/rosetta-sdk-go v0.5.3-0.20201015002757-978c228143a3 + github.com/coinbase/rosetta-sdk-go v0.5.3 github.com/fatih/color v1.9.0 github.com/olekukonko/tablewriter v0.0.2-0.20190409134802-7e037d187b0c github.com/spf13/cobra v1.1.0 diff --git a/go.sum b/go.sum index 7a6f0a75..7e847b32 100644 --- a/go.sum +++ b/go.sum @@ -82,6 +82,8 @@ github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDk github.com/cloudflare/cloudflare-go v0.10.2-0.20190916151808-a80f83b9add9/go.mod h1:1MxXX1Ux4x6mqPmjkUgTP1CdXIBXKX7T+Jk9Gxrmx+U= github.com/coinbase/rosetta-sdk-go v0.5.3-0.20201015002757-978c228143a3 h1:p8p0dVZDh/Zs35tzQq3nFftgFbphNvSzZ6dzLT31F7I= github.com/coinbase/rosetta-sdk-go v0.5.3-0.20201015002757-978c228143a3/go.mod h1:QVVeKHWFNb0NyzEY06LxXMAylJkYa7n+Hk03pORr0ws= +github.com/coinbase/rosetta-sdk-go v0.5.3 h1:AujxZ2VDbq80x7SJDHd1nKOYA8AFD5H27gx07uerFpw= +github.com/coinbase/rosetta-sdk-go v0.5.3/go.mod h1:QVVeKHWFNb0NyzEY06LxXMAylJkYa7n+Hk03pORr0ws= github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= From fad742d3794029945414846f1d386c497836cab8 Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Thu, 15 Oct 2020 11:37:25 -0700 Subject: [PATCH 114/376] Update rosetta-sdk-go to support dynamic pruning --- go.mod | 2 +- go.sum | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index be91211f..affa6be4 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/coinbase/rosetta-cli go 1.13 require ( - github.com/coinbase/rosetta-sdk-go v0.5.3 + github.com/coinbase/rosetta-sdk-go v0.5.4-0.20201015182847-250f7d0adf51 github.com/fatih/color v1.9.0 github.com/olekukonko/tablewriter v0.0.2-0.20190409134802-7e037d187b0c github.com/spf13/cobra v1.1.0 diff --git a/go.sum b/go.sum index 7e847b32..da6c946d 100644 --- a/go.sum +++ b/go.sum @@ -84,6 +84,8 @@ github.com/coinbase/rosetta-sdk-go v0.5.3-0.20201015002757-978c228143a3 h1:p8p0d github.com/coinbase/rosetta-sdk-go v0.5.3-0.20201015002757-978c228143a3/go.mod h1:QVVeKHWFNb0NyzEY06LxXMAylJkYa7n+Hk03pORr0ws= github.com/coinbase/rosetta-sdk-go v0.5.3 h1:AujxZ2VDbq80x7SJDHd1nKOYA8AFD5H27gx07uerFpw= github.com/coinbase/rosetta-sdk-go v0.5.3/go.mod h1:QVVeKHWFNb0NyzEY06LxXMAylJkYa7n+Hk03pORr0ws= +github.com/coinbase/rosetta-sdk-go v0.5.4-0.20201015182847-250f7d0adf51 h1:hcSMD3TDZF4VMJOU2395L60CFgR8JOdtIT4vb6B97nQ= +github.com/coinbase/rosetta-sdk-go v0.5.4-0.20201015182847-250f7d0adf51/go.mod h1:QVVeKHWFNb0NyzEY06LxXMAylJkYa7n+Hk03pORr0ws= github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= From a6edb5a586297648e8b7a2866e020d19ae5db8ee Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Thu, 15 Oct 2020 11:46:27 -0700 Subject: [PATCH 115/376] Implement custom pruning --- pkg/tester/data.go | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/pkg/tester/data.go b/pkg/tester/data.go index dbbf05a0..43777662 100644 --- a/pkg/tester/data.go +++ b/pkg/tester/data.go @@ -64,7 +64,8 @@ const ( EndAtTipCheckInterval = 10 * time.Second ) -var _ http.Handler = (*ConstructionTester)(nil) +var _ http.Handler = (*DataTester)(nil) +var _ statefulsyncer.PruneHelper = (*DataTester)(nil) // DataTester coordinates the `check:data` test. type DataTester struct { @@ -325,7 +326,21 @@ func (t *DataTester) StartPruning( return nil } - return t.syncer.Prune(ctx, statefulsyncer.DefaultPruningDepth) + return t.syncer.Prune(ctx, t) +} + +// PruneableIndex is the index that is +// safe for pruning. +func (t *DataTester) PruneableIndex( + ctx context.Context, + headIndex int64, +) (int64, error) { + if t.config.Data.ReconciliationDisabled { + // It is ok if the returned value here is negative + return headIndex - statefulsyncer.DefaultPruningDepth, nil + } + + return t.reconciler.LastIndexReconciled() - 1, nil } // StartReconciler starts the reconciler if From 0a2b0d732ae07d9d1b4e1129ce7abe9a8df80938 Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Thu, 15 Oct 2020 11:55:25 -0700 Subject: [PATCH 116/376] Don't print if stats are nil --- pkg/results/construction_results.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/pkg/results/construction_results.go b/pkg/results/construction_results.go index 04ec06fa..83e44135 100644 --- a/pkg/results/construction_results.go +++ b/pkg/results/construction_results.go @@ -51,8 +51,10 @@ func (c *CheckConstructionResults) Print() { } fmt.Printf("\n") - c.Stats.Print() - fmt.Printf("\n") + if c.Stats != nil { + c.Stats.Print() + fmt.Printf("\n") + } } // Output writes CheckConstructionResults to the provided From 2109edf0946159a8c4eef2212676973ffa4ae65d Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Thu, 15 Oct 2020 12:01:41 -0700 Subject: [PATCH 117/376] Support compression disabled --- configuration/types.go | 6 ++++++ pkg/tester/construction.go | 7 ++++++- pkg/tester/data.go | 7 ++++++- 3 files changed, 18 insertions(+), 2 deletions(-) diff --git a/configuration/types.go b/configuration/types.go index d0ec97c8..ae454826 100644 --- a/configuration/types.go +++ b/configuration/types.go @@ -341,6 +341,12 @@ type Configuration struct { // should be printed to the console when a file is loaded. LogConfiguration bool `json:"log_configuration"` + // CompressionDisabled configures the storage layer to not + // perform data compression before writing to disk. This leads + // to significantly more on-disk storage usage but can lead + // to performance gains. + CompressionDisabled bool `json:"compression_disabled"` + Construction *ConstructionConfiguration `json:"construction"` Data *DataConfiguration `json:"data"` } diff --git a/pkg/tester/construction.go b/pkg/tester/construction.go index 50fff819..01fac688 100644 --- a/pkg/tester/construction.go +++ b/pkg/tester/construction.go @@ -84,7 +84,12 @@ func InitializeConstruction( log.Fatalf("%s: cannot create command path", err.Error()) } - localStore, err := storage.NewBadgerStorage(ctx, dataPath) + opts := []storage.BadgerOption{} + if config.CompressionDisabled { + opts = append(opts, storage.WithoutCompression()) + } + + localStore, err := storage.NewBadgerStorage(ctx, dataPath, opts...) if err != nil { log.Fatalf("%s: unable to initialize database", err.Error()) } diff --git a/pkg/tester/data.go b/pkg/tester/data.go index 43777662..3c15176a 100644 --- a/pkg/tester/data.go +++ b/pkg/tester/data.go @@ -146,7 +146,12 @@ func InitializeData( log.Fatalf("%s: cannot create command path", err.Error()) } - localStore, err := storage.NewBadgerStorage(ctx, dataPath) + opts := []storage.BadgerOption{} + if config.CompressionDisabled { + opts = append(opts, storage.WithoutCompression()) + } + + localStore, err := storage.NewBadgerStorage(ctx, dataPath, opts...) if err != nil { log.Fatalf("%s: unable to initialize database", err.Error()) } From 2f09bfff782fd26348b69b15c2cc1b77fe195f22 Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Thu, 15 Oct 2020 13:09:20 -0700 Subject: [PATCH 118/376] Use new performance mode --- configuration/types.go | 5 +++++ pkg/tester/construction.go | 3 +++ pkg/tester/data.go | 3 +++ 3 files changed, 11 insertions(+) diff --git a/configuration/types.go b/configuration/types.go index ae454826..90c79031 100644 --- a/configuration/types.go +++ b/configuration/types.go @@ -347,6 +347,11 @@ type Configuration struct { // to performance gains. CompressionDisabled bool `json:"compression_disabled"` + // MemoryLimitDisabled configures storage to increase memory + // usage. Enabling this massively increases performance + // but can use 10s of GBs of RAM, even with pruning enabled. + MemoryLimitDisabled bool `json:"memory_limit_disabled"` + Construction *ConstructionConfiguration `json:"construction"` Data *DataConfiguration `json:"data"` } diff --git a/pkg/tester/construction.go b/pkg/tester/construction.go index 01fac688..93c8df73 100644 --- a/pkg/tester/construction.go +++ b/pkg/tester/construction.go @@ -88,6 +88,9 @@ func InitializeConstruction( if config.CompressionDisabled { opts = append(opts, storage.WithoutCompression()) } + if config.MemoryLimitDisabled { + opts = append(opts, storage.WithCustomSettings(storage.PerformanceBadgerOptions(dataPath))) + } localStore, err := storage.NewBadgerStorage(ctx, dataPath, opts...) if err != nil { diff --git a/pkg/tester/data.go b/pkg/tester/data.go index 3c15176a..4bdb1d1b 100644 --- a/pkg/tester/data.go +++ b/pkg/tester/data.go @@ -150,6 +150,9 @@ func InitializeData( if config.CompressionDisabled { opts = append(opts, storage.WithoutCompression()) } + if config.MemoryLimitDisabled { + opts = append(opts, storage.WithCustomSettings(storage.PerformanceBadgerOptions(dataPath))) + } localStore, err := storage.NewBadgerStorage(ctx, dataPath, opts...) if err != nil { From 355d5b43769dbc15fdb9e142cc52e23eeee2378d Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Thu, 15 Oct 2020 13:10:14 -0700 Subject: [PATCH 119/376] Update rosetta-sdk-go --- go.mod | 2 +- go.sum | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index affa6be4..82218dc1 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/coinbase/rosetta-cli go 1.13 require ( - github.com/coinbase/rosetta-sdk-go v0.5.4-0.20201015182847-250f7d0adf51 + github.com/coinbase/rosetta-sdk-go v0.5.4 github.com/fatih/color v1.9.0 github.com/olekukonko/tablewriter v0.0.2-0.20190409134802-7e037d187b0c github.com/spf13/cobra v1.1.0 diff --git a/go.sum b/go.sum index da6c946d..07c43ab5 100644 --- a/go.sum +++ b/go.sum @@ -86,6 +86,8 @@ github.com/coinbase/rosetta-sdk-go v0.5.3 h1:AujxZ2VDbq80x7SJDHd1nKOYA8AFD5H27gx github.com/coinbase/rosetta-sdk-go v0.5.3/go.mod h1:QVVeKHWFNb0NyzEY06LxXMAylJkYa7n+Hk03pORr0ws= github.com/coinbase/rosetta-sdk-go v0.5.4-0.20201015182847-250f7d0adf51 h1:hcSMD3TDZF4VMJOU2395L60CFgR8JOdtIT4vb6B97nQ= github.com/coinbase/rosetta-sdk-go v0.5.4-0.20201015182847-250f7d0adf51/go.mod h1:QVVeKHWFNb0NyzEY06LxXMAylJkYa7n+Hk03pORr0ws= +github.com/coinbase/rosetta-sdk-go v0.5.4 h1:pM18LK2ci8zZwIu+uETmP6BXHqZbQGXRulwQCjYudg4= +github.com/coinbase/rosetta-sdk-go v0.5.4/go.mod h1:QVVeKHWFNb0NyzEY06LxXMAylJkYa7n+Hk03pORr0ws= github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= From 7af59c5c486f846b32a860f335f201726a9d180e Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Thu, 15 Oct 2020 13:14:36 -0700 Subject: [PATCH 120/376] Update default config example --- examples/configuration/default.json | 2 ++ 1 file changed, 2 insertions(+) diff --git a/examples/configuration/default.json b/examples/configuration/default.json index 46869fe8..91b82182 100644 --- a/examples/configuration/default.json +++ b/examples/configuration/default.json @@ -12,6 +12,8 @@ "max_sync_concurrency": 64, "tip_delay": 300, "log_configuration": false, + "compression_disabled": false, + "memory_limit_disabled": false, "construction": null, "data": { "active_reconciliation_concurrency": 16, From 25bfc580103604eeb8d6dce6b9aebfd0eb4353e8 Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Fri, 16 Oct 2020 09:29:15 -0700 Subject: [PATCH 121/376] Update rosetta-sdk-go --- go.mod | 2 +- go.sum | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index 82218dc1..65626fc9 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/coinbase/rosetta-cli go 1.13 require ( - github.com/coinbase/rosetta-sdk-go v0.5.4 + github.com/coinbase/rosetta-sdk-go v0.5.5 github.com/fatih/color v1.9.0 github.com/olekukonko/tablewriter v0.0.2-0.20190409134802-7e037d187b0c github.com/spf13/cobra v1.1.0 diff --git a/go.sum b/go.sum index 07c43ab5..7e4bf106 100644 --- a/go.sum +++ b/go.sum @@ -88,6 +88,8 @@ github.com/coinbase/rosetta-sdk-go v0.5.4-0.20201015182847-250f7d0adf51 h1:hcSMD github.com/coinbase/rosetta-sdk-go v0.5.4-0.20201015182847-250f7d0adf51/go.mod h1:QVVeKHWFNb0NyzEY06LxXMAylJkYa7n+Hk03pORr0ws= github.com/coinbase/rosetta-sdk-go v0.5.4 h1:pM18LK2ci8zZwIu+uETmP6BXHqZbQGXRulwQCjYudg4= github.com/coinbase/rosetta-sdk-go v0.5.4/go.mod h1:QVVeKHWFNb0NyzEY06LxXMAylJkYa7n+Hk03pORr0ws= +github.com/coinbase/rosetta-sdk-go v0.5.5 h1:Z61/VUO89BDVl1m6Zj0e6OWMl5GnVevj4z79Eh3sSL0= +github.com/coinbase/rosetta-sdk-go v0.5.5/go.mod h1:JRO4BJjhWAI7nYGwzYZWFgYfCTsQOvdZB7tGyN6kEtE= github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= @@ -133,6 +135,7 @@ github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.m github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/ethereum/go-ethereum v1.9.22 h1:/Fea9n2EWJuNJ9oahMq9luqjRBcbW7QWdThbcJl13ek= github.com/ethereum/go-ethereum v1.9.22/go.mod h1:FQjK3ZwD8C5DYn7ukTmFee36rq1dOMESiUfXr5RUc1w= +github.com/ethereum/go-ethereum v1.9.23/go.mod h1:JIfVb6esrqALTExdz9hRYvrP0xBDf6wCncIu1hNwHpM= github.com/fatih/color v1.3.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/fatih/color v1.9.0 h1:8xPHl4/q1VyqGIPif1F+1V3Y3lSmrq01EabUW3CoW5s= From 9c540f7f117421062dc9a8f27aadf773b5cd072b Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Fri, 16 Oct 2020 09:31:45 -0700 Subject: [PATCH 122/376] Use canonical block method --- go.sum | 1 + pkg/processor/reconciler_helper.go | 18 ++++-------------- pkg/tester/data.go | 11 +++++------ 3 files changed, 10 insertions(+), 20 deletions(-) diff --git a/go.sum b/go.sum index 7e4bf106..285c5299 100644 --- a/go.sum +++ b/go.sum @@ -135,6 +135,7 @@ github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.m github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/ethereum/go-ethereum v1.9.22 h1:/Fea9n2EWJuNJ9oahMq9luqjRBcbW7QWdThbcJl13ek= github.com/ethereum/go-ethereum v1.9.22/go.mod h1:FQjK3ZwD8C5DYn7ukTmFee36rq1dOMESiUfXr5RUc1w= +github.com/ethereum/go-ethereum v1.9.23 h1:SIKhg/z4Q7AbvqcxuPYvMxf36che/Rq/Pp0IdYEkbtw= github.com/ethereum/go-ethereum v1.9.23/go.mod h1:JIfVb6esrqALTExdz9hRYvrP0xBDf6wCncIu1hNwHpM= github.com/fatih/color v1.3.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= diff --git a/pkg/processor/reconciler_helper.go b/pkg/processor/reconciler_helper.go index 0b5658c3..62d6fdbd 100644 --- a/pkg/processor/reconciler_helper.go +++ b/pkg/processor/reconciler_helper.go @@ -16,7 +16,6 @@ package processor import ( "context" - "errors" "github.com/coinbase/rosetta-sdk-go/fetcher" "github.com/coinbase/rosetta-sdk-go/reconciler" @@ -52,24 +51,15 @@ func NewReconcilerHelper( } } -// BlockExists returns a boolean indicating if block_storage -// contains a block. This is necessary to reconcile across +// CanonicalBlock returns a boolean indicating if a block +// is in the canonical chain. This is necessary to reconcile across // reorgs. If the block returned on an account balance fetch // does not exist, reconciliation will be skipped. -func (h *ReconcilerHelper) BlockExists( +func (h *ReconcilerHelper) CanonicalBlock( ctx context.Context, block *types.BlockIdentifier, ) (bool, error) { - _, err := h.blockStorage.GetBlock(ctx, types.ConstructPartialBlockIdentifier(block)) - if err == nil { - return true, nil - } - - if errors.Is(err, storage.ErrBlockNotFound) { - return false, nil - } - - return false, err + return h.blockStorage.CanonicalBlock(ctx, block) } // CurrentBlock returns the last processed block and is used diff --git a/pkg/tester/data.go b/pkg/tester/data.go index 4bdb1d1b..7d930479 100644 --- a/pkg/tester/data.go +++ b/pkg/tester/data.go @@ -343,12 +343,11 @@ func (t *DataTester) PruneableIndex( ctx context.Context, headIndex int64, ) (int64, error) { - if t.config.Data.ReconciliationDisabled { - // It is ok if the returned value here is negative - return headIndex - statefulsyncer.DefaultPruningDepth, nil - } - - return t.reconciler.LastIndexReconciled() - 1, nil + // We don't need blocks to exist to reconcile + // balances at their index. + // + // It is ok if the returned value here is negative. + return headIndex - statefulsyncer.DefaultPruningDepth, nil } // StartReconciler starts the reconciler if From 36e46996fd8a794b7f134cf555887cc33d69d1d5 Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Fri, 16 Oct 2020 09:32:33 -0700 Subject: [PATCH 123/376] Update version --- cmd/root.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/root.go b/cmd/root.go index d276a4dd..782c1cce 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -247,6 +247,6 @@ var versionCmd = &cobra.Command{ Use: "version", Short: "Print rosetta-cli version", Run: func(cmd *cobra.Command, args []string) { - fmt.Println("v0.5.13") + fmt.Println("v0.5.14") }, } From 54c8f1f63fdebced474b40c9d47453bc573ea732 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 19 Oct 2020 05:32:43 +0000 Subject: [PATCH 124/376] Bump github.com/spf13/cobra from 1.1.0 to 1.1.1 Bumps [github.com/spf13/cobra](https://github.com/spf13/cobra) from 1.1.0 to 1.1.1. - [Release notes](https://github.com/spf13/cobra/releases) - [Changelog](https://github.com/spf13/cobra/blob/master/CHANGELOG.md) - [Commits](https://github.com/spf13/cobra/compare/v1.1.0...v1.1.1) Signed-off-by: dependabot[bot] --- go.mod | 2 +- go.sum | 15 +++------------ 2 files changed, 4 insertions(+), 13 deletions(-) diff --git a/go.mod b/go.mod index 65626fc9..c3777a80 100644 --- a/go.mod +++ b/go.mod @@ -6,7 +6,7 @@ require ( github.com/coinbase/rosetta-sdk-go v0.5.5 github.com/fatih/color v1.9.0 github.com/olekukonko/tablewriter v0.0.2-0.20190409134802-7e037d187b0c - github.com/spf13/cobra v1.1.0 + github.com/spf13/cobra v1.1.1 github.com/stretchr/testify v1.6.1 golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208 ) diff --git a/go.sum b/go.sum index 285c5299..f54bd3bc 100644 --- a/go.sum +++ b/go.sum @@ -80,14 +80,6 @@ github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+ github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cloudflare/cloudflare-go v0.10.2-0.20190916151808-a80f83b9add9/go.mod h1:1MxXX1Ux4x6mqPmjkUgTP1CdXIBXKX7T+Jk9Gxrmx+U= -github.com/coinbase/rosetta-sdk-go v0.5.3-0.20201015002757-978c228143a3 h1:p8p0dVZDh/Zs35tzQq3nFftgFbphNvSzZ6dzLT31F7I= -github.com/coinbase/rosetta-sdk-go v0.5.3-0.20201015002757-978c228143a3/go.mod h1:QVVeKHWFNb0NyzEY06LxXMAylJkYa7n+Hk03pORr0ws= -github.com/coinbase/rosetta-sdk-go v0.5.3 h1:AujxZ2VDbq80x7SJDHd1nKOYA8AFD5H27gx07uerFpw= -github.com/coinbase/rosetta-sdk-go v0.5.3/go.mod h1:QVVeKHWFNb0NyzEY06LxXMAylJkYa7n+Hk03pORr0ws= -github.com/coinbase/rosetta-sdk-go v0.5.4-0.20201015182847-250f7d0adf51 h1:hcSMD3TDZF4VMJOU2395L60CFgR8JOdtIT4vb6B97nQ= -github.com/coinbase/rosetta-sdk-go v0.5.4-0.20201015182847-250f7d0adf51/go.mod h1:QVVeKHWFNb0NyzEY06LxXMAylJkYa7n+Hk03pORr0ws= -github.com/coinbase/rosetta-sdk-go v0.5.4 h1:pM18LK2ci8zZwIu+uETmP6BXHqZbQGXRulwQCjYudg4= -github.com/coinbase/rosetta-sdk-go v0.5.4/go.mod h1:QVVeKHWFNb0NyzEY06LxXMAylJkYa7n+Hk03pORr0ws= github.com/coinbase/rosetta-sdk-go v0.5.5 h1:Z61/VUO89BDVl1m6Zj0e6OWMl5GnVevj4z79Eh3sSL0= github.com/coinbase/rosetta-sdk-go v0.5.5/go.mod h1:JRO4BJjhWAI7nYGwzYZWFgYfCTsQOvdZB7tGyN6kEtE= github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= @@ -133,8 +125,6 @@ github.com/dvyukov/go-fuzz v0.0.0-20200318091601-be3528f3a813/go.mod h1:11Gm+ccJ github.com/edsrzf/mmap-go v0.0.0-20160512033002-935e0e8a636c/go.mod h1:YO35OhQPt3KJa3ryjFM5Bs14WD66h8eGKpfaBNrHW5M= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= -github.com/ethereum/go-ethereum v1.9.22 h1:/Fea9n2EWJuNJ9oahMq9luqjRBcbW7QWdThbcJl13ek= -github.com/ethereum/go-ethereum v1.9.22/go.mod h1:FQjK3ZwD8C5DYn7ukTmFee36rq1dOMESiUfXr5RUc1w= github.com/ethereum/go-ethereum v1.9.23 h1:SIKhg/z4Q7AbvqcxuPYvMxf36che/Rq/Pp0IdYEkbtw= github.com/ethereum/go-ethereum v1.9.23/go.mod h1:JIfVb6esrqALTExdz9hRYvrP0xBDf6wCncIu1hNwHpM= github.com/fatih/color v1.3.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= @@ -377,8 +367,8 @@ github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= github.com/spf13/cobra v0.0.5 h1:f0B+LkLX6DtmRH1isoNA9VTtNUK9K8xYd28JNNfOv/s= github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU= -github.com/spf13/cobra v1.1.0 h1:aq3wCKjTPmzcNWLVGnsFVN4rflK7Uzn10F8/aw8MhdQ= -github.com/spf13/cobra v1.1.0/go.mod h1:yk5b0mALVusDL5fMM6Rd1wgnoO5jUPhwsQ6LQAJTidQ= +github.com/spf13/cobra v1.1.1 h1:KfztREH0tPxJJ+geloSLaAkaPkr4ki2Er5quFV1TDo4= +github.com/spf13/cobra v1.1.1/go.mod h1:WnodtKOvamDL/PwE2M4iKs8aMDBZ5Q5klgD3qfVJQMI= github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= github.com/spf13/pflag v1.0.3 h1:zPAT6CGy6wXeQ7NtTnaTerfKOsV6V6F8agHXFiazDkg= github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= @@ -653,6 +643,7 @@ gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= From 537076df0c849bbbe567a36fb28061aa468553cb Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Mon, 19 Oct 2020 09:52:31 -0700 Subject: [PATCH 125/376] Add config to disable reconciliation drain --- configuration/types.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/configuration/types.go b/configuration/types.go index 90c79031..608cba67 100644 --- a/configuration/types.go +++ b/configuration/types.go @@ -258,6 +258,11 @@ type DataConfiguration struct { // some of the more advanced checks to confirm syncing is working as expected. ReconciliationDisabled bool `json:"reconciliation_disabled"` + // ReconciliationDrainDisabled is a boolean that configures the rosetta-cli + // to exit check:data before the entire active reconciliation queue has + // been drained (if reconciliation is enabled). + ReconciliationDrainDisabled bool `json:"reconciliation_drain_disabled"` + // InactiveDiscrepencySearchDisabled is a boolean indicating if a search // should be performed to find any inactive reconciliation discrepencies. // Note, a search will never be performed if historical balance lookup From 0ef640f3423fb32c42722bfb61ab6ca17abc8c21 Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Mon, 19 Oct 2020 09:55:57 -0700 Subject: [PATCH 126/376] Update default config file --- examples/configuration/default.json | 1 + 1 file changed, 1 insertion(+) diff --git a/examples/configuration/default.json b/examples/configuration/default.json index 91b82182..8e6692cc 100644 --- a/examples/configuration/default.json +++ b/examples/configuration/default.json @@ -28,6 +28,7 @@ "bootstrap_balances": "", "interesting_accounts": "", "reconciliation_disabled": false, + "reconciliation_drain_disabled": false, "inactive_discrepency_search_disabled": false, "balance_tracking_disabled": false, "coin_tracking_disabled": false, From 5bb989797a72094404d81865a8f784e41848c250 Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Mon, 19 Oct 2020 09:56:27 -0700 Subject: [PATCH 127/376] Update version --- cmd/root.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/root.go b/cmd/root.go index 782c1cce..92a27125 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -247,6 +247,6 @@ var versionCmd = &cobra.Command{ Use: "version", Short: "Print rosetta-cli version", Run: func(cmd *cobra.Command, args []string) { - fmt.Println("v0.5.14") + fmt.Println("v0.5.15") }, } From 10909f4b6aacdcd7f92c1dea0f280332ded1efd9 Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Mon, 19 Oct 2020 10:39:04 -0700 Subject: [PATCH 128/376] First pass at cleaning up reconciler queue --- pkg/tester/data.go | 77 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 77 insertions(+) diff --git a/pkg/tester/data.go b/pkg/tester/data.go index 7d930479..fb7ce5e4 100644 --- a/pkg/tester/data.go +++ b/pkg/tester/data.go @@ -617,6 +617,74 @@ func (t *DataTester) WatchEndConditions( return nil } +// EndAtEmptyQueue exits once the active reconciler +// queue is empty and all reconciler goroutines are idle. +func (t *DataTester) EndAtEmptyQueue( + ctx context.Context, +) error { + // TODO: how do we know when reconciler threads are not busy? Otherwise we will cutoff + // when reconciling + + // TODO: what happens if we are stuck behind a high water mark with a queue? + + tc := time.NewTicker(EndAtTipCheckInterval) + defer tc.Stop() + + for { + select { + case <-ctx.Done(): + return ctx.Err() + + case <-tc.C: + queueSize := t.reconciler.QueueSize() + if queueSize == 0 { + t.cancel() + return nil + } + + color.Cyan(fmt.Sprintf( + "Remaining Queue Size: %d", + queueSize, + // TODO: add busy threads/in-progress reconciliations + )) + } + } +} + +// DrainReconcilerQueue returns once the reconciler queue has been drained +// or an error is encountered. +func (t *DataTester) DrainReconcilerQueue(ctx context.Context, sigListeners *[]context.CancelFunc) error { + color.Cyan("Draining reconciler backlog...hold tight") + + // To cancel all execution, need to call multiple cancel functions. + ctx, cancel := context.WithCancel(ctx) + t.cancel = cancel + *sigListeners = append(*sigListeners, cancel) + + // Disable inactive lookups + t.config.Data.InactiveReconciliationConcurrency = 0 + g, ctx := errgroup.WithContext(ctx) + g.Go(func() error { + return t.StartReconciler(ctx) + }) + g.Go(func() error { + return t.EndAtEmptyQueue(ctx) + }) + + err := g.Wait() + + if *t.signalReceived { + return errors.New("reconcilier queue drain halted") + } + + if errors.Is(err, context.Canceled) { + color.Cyan("Drained reconciler backlog!") + return nil + } + + return err +} + // HandleErr is called when `check:data` returns an error. // If historical balance lookups are enabled, HandleErr will attempt to // automatically find any missing balance-changing operations. @@ -643,6 +711,15 @@ func (t *DataTester) HandleErr(ctx context.Context, err error, sigListeners *[]c } if len(t.endCondition) != 0 { + // Wait for reconciliation queue to drain (only if end condition reached) + if !t.config.Data.ReconciliationDrainDisabled && + shouldReconcile(t.config) && + t.reconciler.QueueSize() > 0 { + if drainErr := t.DrainReconcilerQueue(ctx, sigListeners); drainErr != nil { + color.Red("Draining reconciler queue failed: %s", drainErr.Error()) + } + } + return results.ExitData( t.config, t.counterStorage, From 9e7028cc0684c8822f82103ca48cbbe6b4d271c8 Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Mon, 19 Oct 2020 10:40:30 -0700 Subject: [PATCH 129/376] Move context initialization --- cmd/check_construction.go | 3 +-- cmd/check_data.go | 8 +------- pkg/tester/data.go | 6 +++++- 3 files changed, 7 insertions(+), 10 deletions(-) diff --git a/cmd/check_construction.go b/cmd/check_construction.go index b7183636..c26d13b7 100644 --- a/cmd/check_construction.go +++ b/cmd/check_construction.go @@ -157,6 +157,5 @@ func runCheckConstructionCmd(cmd *cobra.Command, args []string) error { sigListeners := []context.CancelFunc{cancel} go handleSignals(&sigListeners) - err = g.Wait() - return constructionTester.HandleErr(err, &sigListeners) + return constructionTester.HandleErr(g.Wait(), &sigListeners) } diff --git a/cmd/check_data.go b/cmd/check_data.go index 55762c99..d6c2b7fa 100644 --- a/cmd/check_data.go +++ b/cmd/check_data.go @@ -158,13 +158,7 @@ func runCheckDataCmd(cmd *cobra.Command, args []string) error { sigListeners := []context.CancelFunc{cancel} go handleSignals(&sigListeners) - err = g.Wait() - - // Initialize new context because calling context - // will no longer be usable when after termination. - ctx = context.Background() - // HandleErr will exit if we should not attempt // to find missing operations. - return dataTester.HandleErr(ctx, err, &sigListeners) + return dataTester.HandleErr(g.Wait(), &sigListeners) } diff --git a/pkg/tester/data.go b/pkg/tester/data.go index fb7ce5e4..4252dccb 100644 --- a/pkg/tester/data.go +++ b/pkg/tester/data.go @@ -688,7 +688,11 @@ func (t *DataTester) DrainReconcilerQueue(ctx context.Context, sigListeners *[]c // HandleErr is called when `check:data` returns an error. // If historical balance lookups are enabled, HandleErr will attempt to // automatically find any missing balance-changing operations. -func (t *DataTester) HandleErr(ctx context.Context, err error, sigListeners *[]context.CancelFunc) error { +func (t *DataTester) HandleErr(err error, sigListeners *[]context.CancelFunc) error { + // Initialize new context because calling context + // will no longer be usable when after termination. + ctx := context.Background() + if *t.signalReceived { return results.ExitData( t.config, From 8fcef57ef4c199c946fc240ca860279a17a0cffb Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Mon, 19 Oct 2020 20:57:49 -0700 Subject: [PATCH 130/376] Update rosetta-sdk-go --- go.mod | 2 +- go.sum | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index c3777a80..bc97c4c7 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/coinbase/rosetta-cli go 1.13 require ( - github.com/coinbase/rosetta-sdk-go v0.5.5 + github.com/coinbase/rosetta-sdk-go v0.5.6-0.20201020035250-cc90959e74b4 github.com/fatih/color v1.9.0 github.com/olekukonko/tablewriter v0.0.2-0.20190409134802-7e037d187b0c github.com/spf13/cobra v1.1.1 diff --git a/go.sum b/go.sum index f54bd3bc..d9c42831 100644 --- a/go.sum +++ b/go.sum @@ -82,6 +82,8 @@ github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDk github.com/cloudflare/cloudflare-go v0.10.2-0.20190916151808-a80f83b9add9/go.mod h1:1MxXX1Ux4x6mqPmjkUgTP1CdXIBXKX7T+Jk9Gxrmx+U= github.com/coinbase/rosetta-sdk-go v0.5.5 h1:Z61/VUO89BDVl1m6Zj0e6OWMl5GnVevj4z79Eh3sSL0= github.com/coinbase/rosetta-sdk-go v0.5.5/go.mod h1:JRO4BJjhWAI7nYGwzYZWFgYfCTsQOvdZB7tGyN6kEtE= +github.com/coinbase/rosetta-sdk-go v0.5.6-0.20201020035250-cc90959e74b4 h1:fVoxiqoTJyztkXGAxUqeguErNMrvPSpKt3WodMSB4tg= +github.com/coinbase/rosetta-sdk-go v0.5.6-0.20201020035250-cc90959e74b4/go.mod h1:JRO4BJjhWAI7nYGwzYZWFgYfCTsQOvdZB7tGyN6kEtE= github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= From 20240b8116cebb3a657c5312b9c37a4c32f3f38c Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Mon, 19 Oct 2020 21:15:43 -0700 Subject: [PATCH 131/376] Resolve all errors --- pkg/processor/balance_storage_helper.go | 8 ++ pkg/processor/balance_storage_helper_test.go | 2 + pkg/processor/reconciler_handler.go | 2 + pkg/tester/construction.go | 8 +- pkg/tester/data.go | 78 ++++++++++++++++---- 5 files changed, 83 insertions(+), 15 deletions(-) diff --git a/pkg/processor/balance_storage_helper.go b/pkg/processor/balance_storage_helper.go index b55cbccd..10f2d7e0 100644 --- a/pkg/processor/balance_storage_helper.go +++ b/pkg/processor/balance_storage_helper.go @@ -38,6 +38,7 @@ type BalanceStorageHelper struct { // Configuration settings lookupBalanceByBlock bool exemptAccounts map[string]struct{} + balanceExemptions []*types.BalanceExemption // Interesting-only Parsing interestingOnly bool @@ -51,6 +52,7 @@ func NewBalanceStorageHelper( lookupBalanceByBlock bool, exemptAccounts []*reconciler.AccountCurrency, interestingOnly bool, + balanceExemptions []*types.BalanceExemption, ) *BalanceStorageHelper { exemptMap := map[string]struct{}{} @@ -67,6 +69,7 @@ func NewBalanceStorageHelper( exemptAccounts: exemptMap, interestingAddresses: map[string]struct{}{}, interestingOnly: interestingOnly, + balanceExemptions: balanceExemptions, } } @@ -137,3 +140,8 @@ func (h *BalanceStorageHelper) ExemptFunc() parser.ExemptOperation { return exists } } + +// BalanceExemptions returns a list of *types.BalanceExemption. +func (h *BalanceStorageHelper) BalanceExemptions() []*types.BalanceExemption { + return h.balanceExemptions +} diff --git a/pkg/processor/balance_storage_helper_test.go b/pkg/processor/balance_storage_helper_test.go index f1731537..a2cd7b53 100644 --- a/pkg/processor/balance_storage_helper_test.go +++ b/pkg/processor/balance_storage_helper_test.go @@ -84,6 +84,7 @@ func TestExemptFuncExemptAccounts(t *testing.T) { false, test.exemptAccounts, false, + nil, ) result := helper.ExemptFunc()(&types.Operation{ @@ -125,6 +126,7 @@ func TestExemptFuncInterestingParsing(t *testing.T) { false, nil, true, + nil, ) for _, addr := range test.interestingAddresses { diff --git a/pkg/processor/reconciler_handler.go b/pkg/processor/reconciler_handler.go index 53c0fe60..96a0cfbc 100644 --- a/pkg/processor/reconciler_handler.go +++ b/pkg/processor/reconciler_handler.go @@ -69,6 +69,8 @@ func (h *ReconcilerHandler) ReconciliationFailed( liveBalance string, block *types.BlockIdentifier, ) error { + _, _ = h.counterStorage.Update(ctx, storage.FailedReconciliationCounter, big.NewInt(1)) + err := h.logger.ReconcileFailureStream( ctx, reconciliationType, diff --git a/pkg/tester/construction.go b/pkg/tester/construction.go index 93c8df73..b2c46b16 100644 --- a/pkg/tester/construction.go +++ b/pkg/tester/construction.go @@ -97,6 +97,11 @@ func InitializeConstruction( log.Fatalf("%s: unable to initialize database", err.Error()) } + networkOptions, fetchErr := onlineFetcher.NetworkOptionsRetry(ctx, network, nil) + if err != nil { + log.Fatalf("%s: unable to get network options", fetchErr.Err.Error()) + } + counterStorage := storage.NewCounterStorage(localStore) logger := logger.NewLogger( dataPath, @@ -118,6 +123,7 @@ func InitializeConstruction( false, nil, true, + networkOptions.Allow.BalanceExemptions, ) balanceStorageHandler := processor.NewBalanceStorageHandler( @@ -138,7 +144,7 @@ func InitializeConstruction( config.Construction.BlockBroadcastLimit, ) - parser := parser.New(onlineFetcher.Asserter, nil) + parser := parser.New(onlineFetcher.Asserter, nil, networkOptions.Allow.BalanceExemptions) broadcastHelper := processor.NewBroadcastStorageHelper( blockStorage, onlineFetcher, diff --git a/pkg/tester/data.go b/pkg/tester/data.go index 4252dccb..2731a9f4 100644 --- a/pkg/tester/data.go +++ b/pkg/tester/data.go @@ -30,6 +30,7 @@ import ( "github.com/coinbase/rosetta-cli/pkg/results" "github.com/coinbase/rosetta-sdk-go/fetcher" + "github.com/coinbase/rosetta-sdk-go/parser" "github.com/coinbase/rosetta-sdk-go/reconciler" "github.com/coinbase/rosetta-sdk-go/statefulsyncer" "github.com/coinbase/rosetta-sdk-go/storage" @@ -84,6 +85,7 @@ type DataTester struct { genesisBlock *types.BlockIdentifier cancel context.CancelFunc historicalBalanceEnabled bool + parser *parser.Parser endCondition configuration.CheckDataEndCondition endConditionDetail string @@ -206,6 +208,12 @@ func InitializeData( log.Fatalf("%s: unable to get network options", fetchErr.Err.Error()) } + parser := parser.New( + fetcher.Asserter, + nil, + networkOptions.Allow.BalanceExemptions, + ) + // Determine if we should perform historical balance lookups var historicalBalanceEnabled bool if config.Data.HistoricalBalanceEnabled != nil { @@ -217,6 +225,7 @@ func InitializeData( r := reconciler.New( reconcilerHelper, reconcilerHandler, + parser, reconciler.WithActiveConcurrency(int(config.Data.ActiveReconciliationConcurrency)), reconciler.WithInactiveConcurrency(int(config.Data.InactiveReconciliationConcurrency)), reconciler.WithLookupBalanceByBlock(historicalBalanceEnabled), @@ -224,7 +233,6 @@ func InitializeData( reconciler.WithSeenAccounts(seenAccounts), reconciler.WithDebugLogging(config.Data.LogReconciliations), reconciler.WithInactiveFrequency(int64(config.Data.InactiveReconciliationFrequency)), - reconciler.WithBalanceExemptions(networkOptions.Allow.BalanceExemptions), ) blockWorkers := []storage.BlockWorker{} @@ -235,6 +243,7 @@ func InitializeData( historicalBalanceEnabled, exemptAccounts, false, + networkOptions.Allow.BalanceExemptions, ) balanceStorageHandler := processor.NewBalanceStorageHandler( @@ -302,6 +311,7 @@ func InitializeData( signalReceived: signalReceived, genesisBlock: genesisBlock, historicalBalanceEnabled: historicalBalanceEnabled, + parser: parser, } } @@ -617,15 +627,40 @@ func (t *DataTester) WatchEndConditions( return nil } -// EndAtEmptyQueue exits once the active reconciler +// CompleteReconciliations returns the sum of all failed, exempt, and successful +// reconciliations. +func (t *DataTester) CompleteReconciliations(ctx context.Context) (int64, error) { + activeReconciliations, err := t.counterStorage.Get(ctx, storage.ActiveReconciliationCounter) + if err != nil { + return -1, fmt.Errorf("%w: cannot get active reconciliations counter", err) + } + + exemptReconciliations, err := t.counterStorage.Get(ctx, storage.ExemptReconciliationCounter) + if err != nil { + return -1, fmt.Errorf("%w: cannot get exempt reconciliations counter", err) + } + + failedReconciliations, err := t.counterStorage.Get(ctx, storage.FailedReconciliationCounter) + if err != nil { + return -1, fmt.Errorf("%w: cannot get failed reconciliations counter", err) + } + + return activeReconciliations.Int64() + exemptReconciliations.Int64() + failedReconciliations.Int64(), nil +} + +// WaitForEmptyQueue exits once the active reconciler // queue is empty and all reconciler goroutines are idle. -func (t *DataTester) EndAtEmptyQueue( +func (t *DataTester) WaitForEmptyQueue( ctx context.Context, ) error { - // TODO: how do we know when reconciler threads are not busy? Otherwise we will cutoff - // when reconciling - - // TODO: what happens if we are stuck behind a high water mark with a queue? + // To ensure we don't exit while a reconciliation is ongoing + // (i.e. when queue size is 0 but there are busy threads), + // we keep track of how many reconciliations we must complete + // and only exit when that many reconciliations have been performed. + startingComplete, err := t.CompleteReconciliations(ctx) + if err != nil { + return err + } tc := time.NewTicker(EndAtTipCheckInterval) defer tc.Stop() @@ -636,16 +671,20 @@ func (t *DataTester) EndAtEmptyQueue( return ctx.Err() case <-tc.C: - queueSize := t.reconciler.QueueSize() - if queueSize == 0 { + nowComplete, err := t.CompleteReconciliations(ctx) + if err != nil { + return err + } + + remainingReconciliations := nowComplete - startingComplete + if remainingReconciliations == 0 { t.cancel() return nil } color.Cyan(fmt.Sprintf( - "Remaining Queue Size: %d", - queueSize, - // TODO: add busy threads/in-progress reconciliations + "[PROGRESS] remaining reconciliations: %d", + remainingReconciliations, )) } } @@ -668,7 +707,7 @@ func (t *DataTester) DrainReconcilerQueue(ctx context.Context, sigListeners *[]c return t.StartReconciler(ctx) }) g.Go(func() error { - return t.EndAtEmptyQueue(ctx) + return t.WaitForEmptyQueue(ctx) }) err := g.Wait() @@ -714,13 +753,22 @@ func (t *DataTester) HandleErr(err error, sigListeners *[]context.CancelFunc) er ) } + // End condition will only be populated if there is + // no error. if len(t.endCondition) != 0 { // Wait for reconciliation queue to drain (only if end condition reached) if !t.config.Data.ReconciliationDrainDisabled && shouldReconcile(t.config) && t.reconciler.QueueSize() > 0 { if drainErr := t.DrainReconcilerQueue(ctx, sigListeners); drainErr != nil { - color.Red("Draining reconciler queue failed: %s", drainErr.Error()) + return results.ExitData( + t.config, + t.counterStorage, + t.balanceStorage, + drainErr, + "", + "", + ) } } @@ -872,6 +920,7 @@ func (t *DataTester) recursiveOpSearch( r := reconciler.New( reconcilerHelper, reconcilerHandler, + t.parser, // When using concurrency > 1, we could start looking up balance changes // on multiple blocks at once. This can cause us to return the wrong block @@ -891,6 +940,7 @@ func (t *DataTester) recursiveOpSearch( t.historicalBalanceEnabled, nil, false, + t.parser.BalanceExemptions, ) balanceStorageHandler := processor.NewBalanceStorageHandler( From 12f880092b447b7ab04b26078a4f62ea87a4779b Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Mon, 19 Oct 2020 21:27:41 -0700 Subject: [PATCH 132/376] Get cli to compile --- cmd/view_block.go | 2 +- pkg/results/data_results.go | 55 ++++++++++++++++++++++---------- pkg/results/data_results_test.go | 5 --- 3 files changed, 40 insertions(+), 22 deletions(-) diff --git a/cmd/view_block.go b/cmd/view_block.go index f6c5de5e..4fc3ecde 100644 --- a/cmd/view_block.go +++ b/cmd/view_block.go @@ -98,7 +98,7 @@ func runViewBlockCmd(cmd *cobra.Command, args []string) error { // Print out all balance changes in a given block. This does NOT exempt // any operations/accounts from parsing. - p := parser.New(newFetcher.Asserter, func(*types.Operation) bool { return false }) + p := parser.New(newFetcher.Asserter, func(*types.Operation) bool { return false }, nil) changes, err := p.BalanceChanges(Context, block, false) if err != nil { return fmt.Errorf("%w: unable to calculate balance changes", err) diff --git a/pkg/results/data_results.go b/pkg/results/data_results.go index 99f21045..33e6d118 100644 --- a/pkg/results/data_results.go +++ b/pkg/results/data_results.go @@ -36,6 +36,11 @@ import ( "github.com/olekukonko/tablewriter" ) +var ( + f = false + tr = true +) + // EndCondition contains the type of // end condition and any detail associated // with the stop. @@ -98,6 +103,7 @@ type CheckDataStats struct { ActiveReconciliations int64 `json:"active_reconciliations"` InactiveReconciliations int64 `json:"inactive_reconciliations"` ExemptReconciliations int64 `json:"exempt_reconciliations"` + FailedReconciliations int64 `json:"failed_reconciliations"` ReconciliationCoverage float64 `json:"reconciliation_coverage"` } @@ -140,6 +146,13 @@ func (c *CheckDataStats) Print() { strconv.FormatInt(c.ExemptReconciliations, 10), }, ) + table.Append( + []string{ + "Failed Reconciliations", + "# of reconciliation failures", + strconv.FormatInt(c.FailedReconciliations, 10), + }, + ) table.Append( []string{ "Reconciliation Coverage", @@ -203,6 +216,12 @@ func ComputeCheckDataStats( return nil } + failedReconciliations, err := counters.Get(ctx, storage.FailedReconciliationCounter) + if err != nil { + log.Printf("%s: cannot get failed reconciliations counter", err.Error()) + return nil + } + stats := &CheckDataStats{ Blocks: blocks.Int64(), Orphans: orphans.Int64(), @@ -211,6 +230,7 @@ func ComputeCheckDataStats( ActiveReconciliations: activeReconciliations.Int64(), InactiveReconciliations: inactiveReconciliations.Int64(), ExemptReconciliations: exemptReconciliations.Int64(), + FailedReconciliations: failedReconciliations.Int64(), } if balances != nil { @@ -474,27 +494,20 @@ func BalanceTrackingTest(cfg *configuration.Configuration, err error, operations // if no reconciliation errors were received. func ReconciliationTest( cfg *configuration.Configuration, - err error, reconciliationsPerformed bool, + reconciliationsFailed bool, ) *bool { - relatedErrors := []error{ - ErrReconciliationFailure, - } - reconciliationPass := true - for _, relatedError := range relatedErrors { - if errors.Is(err, relatedError) { - reconciliationPass = false - break - } + if cfg.Data.BalanceTrackingDisabled || + cfg.Data.ReconciliationDisabled || + (!reconciliationsPerformed && !reconciliationsFailed) { + return nil } - if (cfg.Data.BalanceTrackingDisabled || cfg.Data.ReconciliationDisabled || cfg.Data.IgnoreReconciliationError || - !reconciliationsPerformed) && - reconciliationPass { - return nil + if reconciliationsFailed { + return &f } - return &reconciliationPass + return &tr } // ComputeCheckDataTests returns a populated CheckDataTests. @@ -506,6 +519,7 @@ func ComputeCheckDataTests( ) *CheckDataTests { operationsSeen := false reconciliationsPerformed := false + reconciliationsFailed := false blocksSynced := false if counterStorage != nil { blocks, err := counterStorage.Get(ctx, storage.BlockCounter) @@ -538,6 +552,15 @@ func ComputeCheckDataTests( if err == nil && exemptReconciliations.Int64() > 0 { reconciliationsPerformed = true } + + failedReconciliations, err := counterStorage.Get( + ctx, + storage.FailedReconciliationCounter, + ) + if err == nil && failedReconciliations.Int64() > 0 { + reconciliationsPerformed = true + reconciliationsFailed = true + } } return &CheckDataTests{ @@ -545,7 +568,7 @@ func ComputeCheckDataTests( ResponseAssertion: ResponseAssertionTest(err), BlockSyncing: BlockSyncingTest(err, blocksSynced), BalanceTracking: BalanceTrackingTest(cfg, err, operationsSeen), - Reconciliation: ReconciliationTest(cfg, err, reconciliationsPerformed), + Reconciliation: ReconciliationTest(cfg, reconciliationsPerformed, reconciliationsFailed), } } diff --git a/pkg/results/data_results_test.go b/pkg/results/data_results_test.go index 68ff2f6d..6a8d03a7 100644 --- a/pkg/results/data_results_test.go +++ b/pkg/results/data_results_test.go @@ -33,11 +33,6 @@ import ( "github.com/stretchr/testify/assert" ) -var ( - tr = true - f = false -) - func TestComputeCheckDataResults(t *testing.T) { var tests = map[string]struct { cfg *configuration.Configuration From a0e325761abfc0690f18cfead24ad4aeea5d5e6e Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Mon, 19 Oct 2020 21:42:30 -0700 Subject: [PATCH 133/376] Add failed reconciliations --- pkg/results/data_results.go | 7 ++++++- pkg/results/data_results_test.go | 26 ++++++++++++++++++++++++++ 2 files changed, 32 insertions(+), 1 deletion(-) diff --git a/pkg/results/data_results.go b/pkg/results/data_results.go index 33e6d118..91975b50 100644 --- a/pkg/results/data_results.go +++ b/pkg/results/data_results.go @@ -494,9 +494,14 @@ func BalanceTrackingTest(cfg *configuration.Configuration, err error, operations // if no reconciliation errors were received. func ReconciliationTest( cfg *configuration.Configuration, + err error, reconciliationsPerformed bool, reconciliationsFailed bool, ) *bool { + if errors.Is(err, ErrReconciliationFailure) { + return &f + } + if cfg.Data.BalanceTrackingDisabled || cfg.Data.ReconciliationDisabled || (!reconciliationsPerformed && !reconciliationsFailed) { @@ -568,7 +573,7 @@ func ComputeCheckDataTests( ResponseAssertion: ResponseAssertionTest(err), BlockSyncing: BlockSyncingTest(err, blocksSynced), BalanceTracking: BalanceTrackingTest(cfg, err, operationsSeen), - Reconciliation: ReconciliationTest(cfg, reconciliationsPerformed, reconciliationsFailed), + Reconciliation: ReconciliationTest(cfg, err, reconciliationsPerformed, reconciliationsFailed), } } diff --git a/pkg/results/data_results_test.go b/pkg/results/data_results_test.go index 6a8d03a7..9addec77 100644 --- a/pkg/results/data_results_test.go +++ b/pkg/results/data_results_test.go @@ -43,6 +43,7 @@ func TestComputeCheckDataResults(t *testing.T) { operationCount int64 activeReconciliations int64 inactiveReconciliations int64 + reconciliationFailures int64 // balance storage values provideBalanceStorage bool @@ -298,6 +299,24 @@ func TestComputeCheckDataResults(t *testing.T) { }, }, }, + "default configuration, counter storage, reconciliation errors": { + cfg: configuration.DefaultConfiguration(), + err: []error{ErrReconciliationFailure}, + provideCounterStorage: true, + activeReconciliations: 10, + reconciliationFailures: 19, + result: &CheckDataResults{ + Tests: &CheckDataTests{ + RequestResponse: true, + ResponseAssertion: true, + Reconciliation: &f, + }, + Stats: &CheckDataStats{ + ActiveReconciliations: 10, + FailedReconciliations: 19, + }, + }, + }, "default configuration, no storage, unknown errors": { cfg: configuration.DefaultConfiguration(), err: []error{errors.New("unsure how to handle this error")}, @@ -378,6 +397,13 @@ func TestComputeCheckDataResults(t *testing.T) { big.NewInt(test.inactiveReconciliations), ) assert.NoError(t, err) + + _, err = counterStorage.Update( + ctx, + storage.FailedReconciliationCounter, + big.NewInt(test.reconciliationFailures), + ) + assert.NoError(t, err) } var balanceStorage *storage.BalanceStorage From ffb2dca7da5991a3b3ceebb8bd49a04dc36800ab Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Tue, 20 Oct 2020 10:41:07 -0700 Subject: [PATCH 134/376] Count skipped reconciliations --- go.mod | 2 +- go.sum | 6 +-- pkg/logger/logger.go | 3 +- pkg/processor/reconciler_handler.go | 24 +++++++--- pkg/results/data_results.go | 15 +++++++ pkg/tester/data.go | 68 ++++++++++++++++++----------- 6 files changed, 79 insertions(+), 39 deletions(-) diff --git a/go.mod b/go.mod index bc97c4c7..b67cda55 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/coinbase/rosetta-cli go 1.13 require ( - github.com/coinbase/rosetta-sdk-go v0.5.6-0.20201020035250-cc90959e74b4 + github.com/coinbase/rosetta-sdk-go v0.5.6-0.20201020173919-5019c40fa06c github.com/fatih/color v1.9.0 github.com/olekukonko/tablewriter v0.0.2-0.20190409134802-7e037d187b0c github.com/spf13/cobra v1.1.1 diff --git a/go.sum b/go.sum index d9c42831..7790ac22 100644 --- a/go.sum +++ b/go.sum @@ -80,10 +80,8 @@ github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+ github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cloudflare/cloudflare-go v0.10.2-0.20190916151808-a80f83b9add9/go.mod h1:1MxXX1Ux4x6mqPmjkUgTP1CdXIBXKX7T+Jk9Gxrmx+U= -github.com/coinbase/rosetta-sdk-go v0.5.5 h1:Z61/VUO89BDVl1m6Zj0e6OWMl5GnVevj4z79Eh3sSL0= -github.com/coinbase/rosetta-sdk-go v0.5.5/go.mod h1:JRO4BJjhWAI7nYGwzYZWFgYfCTsQOvdZB7tGyN6kEtE= -github.com/coinbase/rosetta-sdk-go v0.5.6-0.20201020035250-cc90959e74b4 h1:fVoxiqoTJyztkXGAxUqeguErNMrvPSpKt3WodMSB4tg= -github.com/coinbase/rosetta-sdk-go v0.5.6-0.20201020035250-cc90959e74b4/go.mod h1:JRO4BJjhWAI7nYGwzYZWFgYfCTsQOvdZB7tGyN6kEtE= +github.com/coinbase/rosetta-sdk-go v0.5.6-0.20201020173919-5019c40fa06c h1:5L1MQzPHgvj1aPscFj+KReRQLdoXfyP78xWkd9MEVqI= +github.com/coinbase/rosetta-sdk-go v0.5.6-0.20201020173919-5019c40fa06c/go.mod h1:JRO4BJjhWAI7nYGwzYZWFgYfCTsQOvdZB7tGyN6kEtE= github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= diff --git a/pkg/logger/logger.go b/pkg/logger/logger.go index c639a2a6..f614271b 100644 --- a/pkg/logger/logger.go +++ b/pkg/logger/logger.go @@ -97,7 +97,7 @@ func (l *Logger) LogDataStatus(ctx context.Context, status *results.CheckDataSta } statsMessage := fmt.Sprintf( - "[STATS] Blocks: %d (Orphaned: %d) Transactions: %d Operations: %d Reconciliations: %d (Inactive: %d, Exempt: %d, Coverage: %f%%)", // nolint:lll + "[STATS] Blocks: %d (Orphaned: %d) Transactions: %d Operations: %d Reconciliations: %d (Inactive: %d, Exempt: %d, Skipped: %d, Coverage: %f%%)", // nolint:lll status.Stats.Blocks, status.Stats.Orphans, status.Stats.Transactions, @@ -105,6 +105,7 @@ func (l *Logger) LogDataStatus(ctx context.Context, status *results.CheckDataSta status.Stats.ActiveReconciliations+status.Stats.InactiveReconciliations, status.Stats.InactiveReconciliations, status.Stats.ExemptReconciliations, + status.Stats.SkippedReconciliations, status.Stats.ReconciliationCoverage*utils.OneHundred, ) diff --git a/pkg/processor/reconciler_handler.go b/pkg/processor/reconciler_handler.go index 96a0cfbc..fa228a65 100644 --- a/pkg/processor/reconciler_handler.go +++ b/pkg/processor/reconciler_handler.go @@ -146,6 +146,19 @@ func (h *ReconcilerHandler) ReconciliationExempt( return nil } +// ReconciliationSkipped is called each time a reconciliation is skipped. +func (h *ReconcilerHandler) ReconciliationSkipped( + ctx context.Context, + reconciliationType string, + account *types.AccountIdentifier, + currency *types.Currency, + cause string, +) error { + _, _ = h.counterStorage.Update(ctx, storage.SkippedReconciliationsCounter, big.NewInt(1)) + + return nil +} + // ReconciliationSucceeded is called each time a reconciliation succeeds. func (h *ReconcilerHandler) ReconciliationSucceeded( ctx context.Context, @@ -156,16 +169,13 @@ func (h *ReconcilerHandler) ReconciliationSucceeded( block *types.BlockIdentifier, ) error { // Update counters + counter := storage.ActiveReconciliationCounter if reconciliationType == reconciler.InactiveReconciliation { - _, _ = h.counterStorage.Update( - ctx, - storage.InactiveReconciliationCounter, - big.NewInt(1), - ) - } else { - _, _ = h.counterStorage.Update(ctx, storage.ActiveReconciliationCounter, big.NewInt(1)) + counter = storage.InactiveReconciliationCounter } + _, _ = h.counterStorage.Update(ctx, counter, big.NewInt(1)) + if err := h.balanceStorage.Reconciled(ctx, account, currency, block); err != nil { return fmt.Errorf("%w: unable to store updated reconciliation", err) } diff --git a/pkg/results/data_results.go b/pkg/results/data_results.go index 91975b50..bba056ec 100644 --- a/pkg/results/data_results.go +++ b/pkg/results/data_results.go @@ -104,6 +104,7 @@ type CheckDataStats struct { InactiveReconciliations int64 `json:"inactive_reconciliations"` ExemptReconciliations int64 `json:"exempt_reconciliations"` FailedReconciliations int64 `json:"failed_reconciliations"` + SkippedReconciliations int64 `json:"skipped_reconciliations"` ReconciliationCoverage float64 `json:"reconciliation_coverage"` } @@ -153,6 +154,13 @@ func (c *CheckDataStats) Print() { strconv.FormatInt(c.FailedReconciliations, 10), }, ) + table.Append( + []string{ + "Skipped Reconciliations", + "# of reconciliations skipped", + strconv.FormatInt(c.SkippedReconciliations, 10), + }, + ) table.Append( []string{ "Reconciliation Coverage", @@ -222,6 +230,12 @@ func ComputeCheckDataStats( return nil } + skippedReconciliations, err := counters.Get(ctx, storage.SkippedReconciliationsCounter) + if err != nil { + log.Printf("%s: cannot get skipped reconciliations counter", err.Error()) + return nil + } + stats := &CheckDataStats{ Blocks: blocks.Int64(), Orphans: orphans.Int64(), @@ -231,6 +245,7 @@ func ComputeCheckDataStats( InactiveReconciliations: inactiveReconciliations.Int64(), ExemptReconciliations: exemptReconciliations.Int64(), FailedReconciliations: failedReconciliations.Int64(), + SkippedReconciliations: skippedReconciliations.Int64(), } if balances != nil { diff --git a/pkg/tester/data.go b/pkg/tester/data.go index 2731a9f4..1d47b183 100644 --- a/pkg/tester/data.go +++ b/pkg/tester/data.go @@ -175,6 +175,25 @@ func InitializeData( blockStorage := storage.NewBlockStorage(localStore) balanceStorage := storage.NewBalanceStorage(localStore) + // Bootstrap balances, if provided. We need to do before initializing + // the reconciler otherwise we won't reconcile bootstrapped accounts + // until rosetta-cli restart. + if len(config.Data.BootstrapBalances) > 0 { + _, err := blockStorage.GetHeadBlockIdentifier(ctx) + if err == storage.ErrHeadBlockNotFound { + err = balanceStorage.BootstrapBalances( + ctx, + config.Data.BootstrapBalances, + genesisBlock, + ) + if err != nil { + log.Fatalf("%s: unable to bootstrap balances", err.Error()) + } + } else { + log.Println("Skipping balance bootstrapping because already started syncing") + } + } + logger := logger.NewLogger( dataPath, config.Data.LogBlocks, @@ -255,23 +274,6 @@ func InitializeData( balanceStorage.Initialize(balanceStorageHelper, balanceStorageHandler) - // Bootstrap balances if provided - if len(config.Data.BootstrapBalances) > 0 { - _, err := blockStorage.GetHeadBlockIdentifier(ctx) - if err == storage.ErrHeadBlockNotFound { - err = balanceStorage.BootstrapBalances( - ctx, - config.Data.BootstrapBalances, - genesisBlock, - ) - if err != nil { - log.Fatalf("%s: unable to bootstrap balances", err.Error()) - } - } else { - log.Println("Skipping balance bootstrapping because already started syncing") - } - } - blockWorkers = append(blockWorkers, balanceStorage) } @@ -645,7 +647,13 @@ func (t *DataTester) CompleteReconciliations(ctx context.Context) (int64, error) return -1, fmt.Errorf("%w: cannot get failed reconciliations counter", err) } - return activeReconciliations.Int64() + exemptReconciliations.Int64() + failedReconciliations.Int64(), nil + skippedReconciliations, err := t.counterStorage.Get(ctx, storage.SkippedReconciliationsCounter) + if err != nil { + return -1, fmt.Errorf("%w: cannot get skipped reconciliations counter", err) + } + + total := activeReconciliations.Int64() + exemptReconciliations.Int64() + failedReconciliations.Int64() + skippedReconciliations.Int64() + return total, nil } // WaitForEmptyQueue exits once the active reconciler @@ -661,10 +669,16 @@ func (t *DataTester) WaitForEmptyQueue( if err != nil { return err } + startingRemaining := t.reconciler.QueueSize() tc := time.NewTicker(EndAtTipCheckInterval) defer tc.Stop() + color.Cyan( + "[PROGRESS] remaining reconciliations: %d", + startingRemaining, + ) + for { select { case <-ctx.Done(): @@ -676,16 +690,17 @@ func (t *DataTester) WaitForEmptyQueue( return err } - remainingReconciliations := nowComplete - startingComplete - if remainingReconciliations == 0 { + completed := nowComplete - startingComplete + remaining := int64(startingRemaining) - completed + if remaining <= 0 { t.cancel() return nil } - color.Cyan(fmt.Sprintf( + color.Cyan( "[PROGRESS] remaining reconciliations: %d", - remainingReconciliations, - )) + remaining, + ) } } } @@ -693,7 +708,7 @@ func (t *DataTester) WaitForEmptyQueue( // DrainReconcilerQueue returns once the reconciler queue has been drained // or an error is encountered. func (t *DataTester) DrainReconcilerQueue(ctx context.Context, sigListeners *[]context.CancelFunc) error { - color.Cyan("Draining reconciler backlog...hold tight") + color.Cyan("draining reconciler backlog (you can disable this in your configuration file)") // To cancel all execution, need to call multiple cancel functions. ctx, cancel := context.WithCancel(ctx) @@ -701,7 +716,8 @@ func (t *DataTester) DrainReconcilerQueue(ctx context.Context, sigListeners *[]c *sigListeners = append(*sigListeners, cancel) // Disable inactive lookups - t.config.Data.InactiveReconciliationConcurrency = 0 + t.reconciler.InactiveConcurrency = 0 + g, ctx := errgroup.WithContext(ctx) g.Go(func() error { return t.StartReconciler(ctx) @@ -717,7 +733,7 @@ func (t *DataTester) DrainReconcilerQueue(ctx context.Context, sigListeners *[]c } if errors.Is(err, context.Canceled) { - color.Cyan("Drained reconciler backlog!") + color.Cyan("drained reconciler backlog") return nil } From a26d60df23c089a8e975a9afc0f48d6e95edd5a0 Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Tue, 20 Oct 2020 10:45:27 -0700 Subject: [PATCH 135/376] Nits --- pkg/results/data_results.go | 2 +- pkg/tester/data.go | 32 +++++++++++++++++++------------- 2 files changed, 20 insertions(+), 14 deletions(-) diff --git a/pkg/results/data_results.go b/pkg/results/data_results.go index bba056ec..807bbaf8 100644 --- a/pkg/results/data_results.go +++ b/pkg/results/data_results.go @@ -531,7 +531,7 @@ func ReconciliationTest( } // ComputeCheckDataTests returns a populated CheckDataTests. -func ComputeCheckDataTests( +func ComputeCheckDataTests( // nolint:gocognit ctx context.Context, cfg *configuration.Configuration, err error, diff --git a/pkg/tester/data.go b/pkg/tester/data.go index 1d47b183..3b91db78 100644 --- a/pkg/tester/data.go +++ b/pkg/tester/data.go @@ -652,8 +652,10 @@ func (t *DataTester) CompleteReconciliations(ctx context.Context) (int64, error) return -1, fmt.Errorf("%w: cannot get skipped reconciliations counter", err) } - total := activeReconciliations.Int64() + exemptReconciliations.Int64() + failedReconciliations.Int64() + skippedReconciliations.Int64() - return total, nil + return activeReconciliations.Int64() + + exemptReconciliations.Int64() + + failedReconciliations.Int64() + + skippedReconciliations.Int64(), nil } // WaitForEmptyQueue exits once the active reconciler @@ -773,18 +775,22 @@ func (t *DataTester) HandleErr(err error, sigListeners *[]context.CancelFunc) er // no error. if len(t.endCondition) != 0 { // Wait for reconciliation queue to drain (only if end condition reached) - if !t.config.Data.ReconciliationDrainDisabled && - shouldReconcile(t.config) && + if shouldReconcile(t.config) && t.reconciler.QueueSize() > 0 { - if drainErr := t.DrainReconcilerQueue(ctx, sigListeners); drainErr != nil { - return results.ExitData( - t.config, - t.counterStorage, - t.balanceStorage, - drainErr, - "", - "", - ) + if t.config.Data.ReconciliationDrainDisabled { + color.Cyan("skipping reconciler backlog drain (you can enable this in your configuration file)") + } else { + drainErr := t.DrainReconcilerQueue(ctx, sigListeners) + if drainErr != nil { + return results.ExitData( + t.config, + t.counterStorage, + t.balanceStorage, + drainErr, + "", + "", + ) + } } } From 500e32020a033e7d32ea75d36eca2d566db950d8 Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Tue, 20 Oct 2020 10:50:21 -0700 Subject: [PATCH 136/376] Update rosetta-sdk-go --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index b67cda55..521935de 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/coinbase/rosetta-cli go 1.13 require ( - github.com/coinbase/rosetta-sdk-go v0.5.6-0.20201020173919-5019c40fa06c + github.com/coinbase/rosetta-sdk-go v0.5.6 github.com/fatih/color v1.9.0 github.com/olekukonko/tablewriter v0.0.2-0.20190409134802-7e037d187b0c github.com/spf13/cobra v1.1.1 diff --git a/go.sum b/go.sum index 7790ac22..f68fd0ab 100644 --- a/go.sum +++ b/go.sum @@ -80,8 +80,8 @@ github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+ github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cloudflare/cloudflare-go v0.10.2-0.20190916151808-a80f83b9add9/go.mod h1:1MxXX1Ux4x6mqPmjkUgTP1CdXIBXKX7T+Jk9Gxrmx+U= -github.com/coinbase/rosetta-sdk-go v0.5.6-0.20201020173919-5019c40fa06c h1:5L1MQzPHgvj1aPscFj+KReRQLdoXfyP78xWkd9MEVqI= -github.com/coinbase/rosetta-sdk-go v0.5.6-0.20201020173919-5019c40fa06c/go.mod h1:JRO4BJjhWAI7nYGwzYZWFgYfCTsQOvdZB7tGyN6kEtE= +github.com/coinbase/rosetta-sdk-go v0.5.6 h1:mgSSw2NsVUBZPgeQegIWazJki0IT8Zd+xrFLS1afzJE= +github.com/coinbase/rosetta-sdk-go v0.5.6/go.mod h1:JRO4BJjhWAI7nYGwzYZWFgYfCTsQOvdZB7tGyN6kEtE= github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= From ecb6376ded5658689cb0066603e56b3fabbe0bcb Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Tue, 20 Oct 2020 11:09:20 -0700 Subject: [PATCH 137/376] nits --- pkg/tester/data.go | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/pkg/tester/data.go b/pkg/tester/data.go index 3b91db78..efe6ce99 100644 --- a/pkg/tester/data.go +++ b/pkg/tester/data.go @@ -180,7 +180,8 @@ func InitializeData( // until rosetta-cli restart. if len(config.Data.BootstrapBalances) > 0 { _, err := blockStorage.GetHeadBlockIdentifier(ctx) - if err == storage.ErrHeadBlockNotFound { + switch { + case err == storage.ErrHeadBlockNotFound: err = balanceStorage.BootstrapBalances( ctx, config.Data.BootstrapBalances, @@ -189,7 +190,9 @@ func InitializeData( if err != nil { log.Fatalf("%s: unable to bootstrap balances", err.Error()) } - } else { + case err != nil: + log.Fatalf("%s: unable to get head block identifier", err.Error()) + default: log.Println("Skipping balance bootstrapping because already started syncing") } } From 803aa8ea9408664e6e7fa6a0ab165ebef8ada349 Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Thu, 22 Oct 2020 09:24:54 -0700 Subject: [PATCH 138/376] Add ability to disable prefetch --- pkg/processor/balance_storage_helper.go | 5 ++++- pkg/processor/balance_storage_helper_test.go | 2 ++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/pkg/processor/balance_storage_helper.go b/pkg/processor/balance_storage_helper.go index 10f2d7e0..d39b4080 100644 --- a/pkg/processor/balance_storage_helper.go +++ b/pkg/processor/balance_storage_helper.go @@ -39,6 +39,7 @@ type BalanceStorageHelper struct { lookupBalanceByBlock bool exemptAccounts map[string]struct{} balanceExemptions []*types.BalanceExemption + preFetchDisabled bool // Interesting-only Parsing interestingOnly bool @@ -53,6 +54,7 @@ func NewBalanceStorageHelper( exemptAccounts []*reconciler.AccountCurrency, interestingOnly bool, balanceExemptions []*types.BalanceExemption, + preFetchDisabled bool, ) *BalanceStorageHelper { exemptMap := map[string]struct{}{} @@ -70,6 +72,7 @@ func NewBalanceStorageHelper( interestingAddresses: map[string]struct{}{}, interestingOnly: interestingOnly, balanceExemptions: balanceExemptions, + preFetchDisabled: preFetchDisabled, } } @@ -83,7 +86,7 @@ func (h *BalanceStorageHelper) AccountBalance( currency *types.Currency, block *types.BlockIdentifier, ) (*types.Amount, error) { - if !h.lookupBalanceByBlock { + if !h.lookupBalanceByBlock || h.preFetchDisabled { return &types.Amount{ Value: "0", Currency: currency, diff --git a/pkg/processor/balance_storage_helper_test.go b/pkg/processor/balance_storage_helper_test.go index a2cd7b53..55aaabae 100644 --- a/pkg/processor/balance_storage_helper_test.go +++ b/pkg/processor/balance_storage_helper_test.go @@ -85,6 +85,7 @@ func TestExemptFuncExemptAccounts(t *testing.T) { test.exemptAccounts, false, nil, + false, ) result := helper.ExemptFunc()(&types.Operation{ @@ -127,6 +128,7 @@ func TestExemptFuncInterestingParsing(t *testing.T) { nil, true, nil, + false, ) for _, addr := range test.interestingAddresses { From abc3c6026a8af18681ab454c47c97fd792d5aa8e Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Thu, 22 Oct 2020 09:36:02 -0700 Subject: [PATCH 139/376] Add initial balance fetch disabled to config --- configuration/types.go | 9 +++++++++ pkg/processor/balance_storage_helper.go | 8 ++++---- pkg/tester/construction.go | 1 + pkg/tester/data.go | 2 ++ 4 files changed, 16 insertions(+), 4 deletions(-) diff --git a/configuration/types.go b/configuration/types.go index 608cba67..54edc4dd 100644 --- a/configuration/types.go +++ b/configuration/types.go @@ -357,6 +357,15 @@ type Configuration struct { // but can use 10s of GBs of RAM, even with pruning enabled. MemoryLimitDisabled bool `json:"memory_limit_disabled"` + // InitialBalanceFetchDisabled configures rosetta-cli + // not to lookup the balance of newly seen accounts at the + // parent block before applying operations. Disabling + // this step can significantly speed up performance + // without impacting validation accuracy (if all genesis + // accounts are provided using bootstrap_balances and + // syncing starts from genesis). + InitialBalanceFetchDisabled bool `json:"inital_balance_fetch_disabled"` + Construction *ConstructionConfiguration `json:"construction"` Data *DataConfiguration `json:"data"` } diff --git a/pkg/processor/balance_storage_helper.go b/pkg/processor/balance_storage_helper.go index d39b4080..99b737ff 100644 --- a/pkg/processor/balance_storage_helper.go +++ b/pkg/processor/balance_storage_helper.go @@ -39,7 +39,7 @@ type BalanceStorageHelper struct { lookupBalanceByBlock bool exemptAccounts map[string]struct{} balanceExemptions []*types.BalanceExemption - preFetchDisabled bool + initialFetchDisabled bool // Interesting-only Parsing interestingOnly bool @@ -54,7 +54,7 @@ func NewBalanceStorageHelper( exemptAccounts []*reconciler.AccountCurrency, interestingOnly bool, balanceExemptions []*types.BalanceExemption, - preFetchDisabled bool, + initialFetchDisabled bool, ) *BalanceStorageHelper { exemptMap := map[string]struct{}{} @@ -72,7 +72,7 @@ func NewBalanceStorageHelper( interestingAddresses: map[string]struct{}{}, interestingOnly: interestingOnly, balanceExemptions: balanceExemptions, - preFetchDisabled: preFetchDisabled, + initialFetchDisabled: initialFetchDisabled, } } @@ -86,7 +86,7 @@ func (h *BalanceStorageHelper) AccountBalance( currency *types.Currency, block *types.BlockIdentifier, ) (*types.Amount, error) { - if !h.lookupBalanceByBlock || h.preFetchDisabled { + if !h.lookupBalanceByBlock || h.initialFetchDisabled { return &types.Amount{ Value: "0", Currency: currency, diff --git a/pkg/tester/construction.go b/pkg/tester/construction.go index b2c46b16..b2c47be8 100644 --- a/pkg/tester/construction.go +++ b/pkg/tester/construction.go @@ -124,6 +124,7 @@ func InitializeConstruction( nil, true, networkOptions.Allow.BalanceExemptions, + config.InitialBalanceFetchDisabled, ) balanceStorageHandler := processor.NewBalanceStorageHandler( diff --git a/pkg/tester/data.go b/pkg/tester/data.go index efe6ce99..623725a8 100644 --- a/pkg/tester/data.go +++ b/pkg/tester/data.go @@ -266,6 +266,7 @@ func InitializeData( exemptAccounts, false, networkOptions.Allow.BalanceExemptions, + config.InitialBalanceFetchDisabled, ) balanceStorageHandler := processor.NewBalanceStorageHandler( @@ -966,6 +967,7 @@ func (t *DataTester) recursiveOpSearch( nil, false, t.parser.BalanceExemptions, + false, // we will need to perform an initial balance fetch when finding issues ) balanceStorageHandler := processor.NewBalanceStorageHandler( From b1fa3229ab76c83ee59b6772019286dc130aa282 Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Thu, 22 Oct 2020 09:36:41 -0700 Subject: [PATCH 140/376] nits --- configuration/types.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configuration/types.go b/configuration/types.go index 54edc4dd..6ea0eea3 100644 --- a/configuration/types.go +++ b/configuration/types.go @@ -364,7 +364,7 @@ type Configuration struct { // without impacting validation accuracy (if all genesis // accounts are provided using bootstrap_balances and // syncing starts from genesis). - InitialBalanceFetchDisabled bool `json:"inital_balance_fetch_disabled"` + InitialBalanceFetchDisabled bool `json:"initial_balance_fetch_disabled"` Construction *ConstructionConfiguration `json:"construction"` Data *DataConfiguration `json:"data"` From ce8c6a8313de8237bac63cee5060442b4edb4d16 Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Thu, 22 Oct 2020 09:46:39 -0700 Subject: [PATCH 141/376] Split configs --- cmd/root.go | 2 +- configuration/types.go | 28 +++++++++++++++++++--------- pkg/tester/construction.go | 2 +- pkg/tester/data.go | 2 +- 4 files changed, 22 insertions(+), 12 deletions(-) diff --git a/cmd/root.go b/cmd/root.go index 92a27125..19d0bc89 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -247,6 +247,6 @@ var versionCmd = &cobra.Command{ Use: "version", Short: "Print rosetta-cli version", Run: func(cmd *cobra.Command, args []string) { - fmt.Println("v0.5.15") + fmt.Println("v0.5.16") }, } diff --git a/configuration/types.go b/configuration/types.go index 6ea0eea3..2349f829 100644 --- a/configuration/types.go +++ b/configuration/types.go @@ -146,6 +146,16 @@ type ConstructionConfiguration struct { // Quiet is a boolean indicating if all request and response // logging should be silenced. Quiet bool `json:"quiet,omitempty"` + + // InitialBalanceFetchDisabled configures rosetta-cli + // not to lookup the balance of newly seen accounts at the + // parent block before applying operations. Disabling this + // is only a good idea if you create multiple new accounts each block + // and don't fund any accounts before starting check:construction. + // + // This is a separate config from the data config because it + // is usually false whereas the data config by the same name is usually true. + InitialBalanceFetchDisabled bool `json:"initial_balance_fetch_disabled"` } // ReconciliationCoverage is used to add conditions @@ -303,6 +313,15 @@ type DataConfiguration struct { // wish to use `start_index` at a later point to restart from some // previously synced block. PruningDisabled bool `json:"pruning_disabled"` + + // InitialBalanceFetchDisabled configures rosetta-cli + // not to lookup the balance of newly seen accounts at the + // parent block before applying operations. Disabling + // this step can significantly speed up performance + // without impacting validation accuracy (if all genesis + // accounts are provided using bootstrap_balances and + // syncing starts from genesis). + InitialBalanceFetchDisabled bool `json:"initial_balance_fetch_disabled"` } // Configuration contains all configuration settings for running @@ -357,15 +376,6 @@ type Configuration struct { // but can use 10s of GBs of RAM, even with pruning enabled. MemoryLimitDisabled bool `json:"memory_limit_disabled"` - // InitialBalanceFetchDisabled configures rosetta-cli - // not to lookup the balance of newly seen accounts at the - // parent block before applying operations. Disabling - // this step can significantly speed up performance - // without impacting validation accuracy (if all genesis - // accounts are provided using bootstrap_balances and - // syncing starts from genesis). - InitialBalanceFetchDisabled bool `json:"initial_balance_fetch_disabled"` - Construction *ConstructionConfiguration `json:"construction"` Data *DataConfiguration `json:"data"` } diff --git a/pkg/tester/construction.go b/pkg/tester/construction.go index b2c47be8..746cbd0e 100644 --- a/pkg/tester/construction.go +++ b/pkg/tester/construction.go @@ -124,7 +124,7 @@ func InitializeConstruction( nil, true, networkOptions.Allow.BalanceExemptions, - config.InitialBalanceFetchDisabled, + config.Construction.InitialBalanceFetchDisabled, ) balanceStorageHandler := processor.NewBalanceStorageHandler( diff --git a/pkg/tester/data.go b/pkg/tester/data.go index 623725a8..68d6f852 100644 --- a/pkg/tester/data.go +++ b/pkg/tester/data.go @@ -266,7 +266,7 @@ func InitializeData( exemptAccounts, false, networkOptions.Allow.BalanceExemptions, - config.InitialBalanceFetchDisabled, + config.Data.InitialBalanceFetchDisabled, ) balanceStorageHandler := processor.NewBalanceStorageHandler( From aad10bd0afe1ec8e3ca2b19b161efe4629669e3d Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Thu, 22 Oct 2020 09:52:32 -0700 Subject: [PATCH 142/376] Update examples --- examples/configuration/default.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/examples/configuration/default.json b/examples/configuration/default.json index 8e6692cc..4fe1a204 100644 --- a/examples/configuration/default.json +++ b/examples/configuration/default.json @@ -34,6 +34,7 @@ "coin_tracking_disabled": false, "status_port": 9090, "results_output_file": "", - "pruning_disabled": false + "pruning_disabled": false, + "initial_balance_fetch_disabled": false } } \ No newline at end of file From 2c7b5ddc5347a7f2df21dab46a2ab2e6ce7da2f0 Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Thu, 22 Oct 2020 10:06:02 -0700 Subject: [PATCH 143/376] Throw error if balance exemptions > 0 but initial balance fetch is disabled --- pkg/tester/construction.go | 4 ++++ pkg/tester/data.go | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/pkg/tester/construction.go b/pkg/tester/construction.go index 746cbd0e..8745fbe9 100644 --- a/pkg/tester/construction.go +++ b/pkg/tester/construction.go @@ -102,6 +102,10 @@ func InitializeConstruction( log.Fatalf("%s: unable to get network options", fetchErr.Err.Error()) } + if len(networkOptions.Allow.BalanceExemptions) > 0 && config.Construction.InitialBalanceFetchDisabled { + log.Fatal("found balance exemptions but initial balance fetch disabled") + } + counterStorage := storage.NewCounterStorage(localStore) logger := logger.NewLogger( dataPath, diff --git a/pkg/tester/data.go b/pkg/tester/data.go index 68d6f852..c4e52215 100644 --- a/pkg/tester/data.go +++ b/pkg/tester/data.go @@ -230,6 +230,10 @@ func InitializeData( log.Fatalf("%s: unable to get network options", fetchErr.Err.Error()) } + if len(networkOptions.Allow.BalanceExemptions) > 0 && config.Data.InitialBalanceFetchDisabled { + log.Fatal("found balance exemptions but initial balance fetch disabled") + } + parser := parser.New( fetcher.Asserter, nil, From 0361e091fdd74963be3f13e01b04437e781f7f97 Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Thu, 22 Oct 2020 17:33:06 -0700 Subject: [PATCH 144/376] Optionally, only print balance changes --- cmd/root.go | 10 ++++++++++ cmd/view_block.go | 31 +++++++++++++++++++++++-------- 2 files changed, 33 insertions(+), 8 deletions(-) diff --git a/cmd/root.go b/cmd/root.go index 19d0bc89..f74756ff 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -62,6 +62,10 @@ var ( // blockProfileCleanup is called after the root command is executed to // cleanup a running block profile. blockProfileCleanup func() + + // OnlyChanges is a boolean indicating if only the balance changes should be + // logged to the console. + OnlyChanges bool ) // rootPreRun is executed before the root command runs and sets up cpu @@ -192,6 +196,12 @@ default values.`, rootCmd.AddCommand(checkConstructionCmd) // View Commands + viewBlockCmd.Flags().BoolVar( + &OnlyChanges, + "only-changes", + false, + `Only print balance changes for accounts in the block`, + ) rootCmd.AddCommand(viewBlockCmd) rootCmd.AddCommand(viewAccountCmd) rootCmd.AddCommand(viewNetworksCmd) diff --git a/cmd/view_block.go b/cmd/view_block.go index 4fc3ecde..aee09b17 100644 --- a/cmd/view_block.go +++ b/cmd/view_block.go @@ -94,26 +94,41 @@ func runViewBlockCmd(cmd *cobra.Command, args []string) error { return fmt.Errorf("%w: unable to fetch block", fetchErr.Err) } - log.Printf("Current Block: %s\n", types.PrettyPrintStruct(block)) + if !OnlyChanges { + log.Printf("Current Block: %s\n", types.PrettyPrintStruct(block)) + } // Print out all balance changes in a given block. This does NOT exempt // any operations/accounts from parsing. p := parser.New(newFetcher.Asserter, func(*types.Operation) bool { return false }, nil) - changes, err := p.BalanceChanges(Context, block, false) + balanceChanges, err := p.BalanceChanges(Context, block, false) if err != nil { return fmt.Errorf("%w: unable to calculate balance changes", err) } - log.Printf("Balance Changes: %s\n", types.PrettyPrintStruct(changes)) + for _, balanceChange := range balanceChanges { + parsedDiff, err := types.BigInt(balanceChange.Difference) + if err != nil { + return fmt.Errorf("%w: unable to parse Difference", err) + } - // Print out all OperationGroups for each transaction in a block. - for _, tx := range block.Transactions { log.Printf( - "Transaction %s Operation Groups: %s\n", - tx.TransactionIdentifier.Hash, - types.PrettyPrintStruct(parser.GroupOperations(tx)), + "%s -> %s\n", + types.PrintStruct(balanceChange.Account), + utils.PrettyAmount(parsedDiff, balanceChange.Currency), ) } + if !OnlyChanges { + // Print out all OperationGroups for each transaction in a block. + for _, tx := range block.Transactions { + log.Printf( + "Transaction %s Operation Groups: %s\n", + tx.TransactionIdentifier.Hash, + types.PrettyPrintStruct(parser.GroupOperations(tx)), + ) + } + } + return nil } From f0b38348edbb427793fc08a52590c44cd83d91c4 Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Thu, 22 Oct 2020 17:44:13 -0700 Subject: [PATCH 145/376] Update version --- cmd/root.go | 2 +- cmd/view_block.go | 44 +++++++++++++++++++++++++++++++++++++++----- 2 files changed, 40 insertions(+), 6 deletions(-) diff --git a/cmd/root.go b/cmd/root.go index f74756ff..a7ee8995 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -257,6 +257,6 @@ var versionCmd = &cobra.Command{ Use: "version", Short: "Print rosetta-cli version", Run: func(cmd *cobra.Command, args []string) { - fmt.Println("v0.5.16") + fmt.Println("v0.5.17") }, } diff --git a/cmd/view_block.go b/cmd/view_block.go index aee09b17..235f1a60 100644 --- a/cmd/view_block.go +++ b/cmd/view_block.go @@ -16,7 +16,6 @@ package cmd import ( "fmt" - "log" "strconv" "time" @@ -95,7 +94,7 @@ func runViewBlockCmd(cmd *cobra.Command, args []string) error { } if !OnlyChanges { - log.Printf("Current Block: %s\n", types.PrettyPrintStruct(block)) + fmt.Println("Current Block:", types.PrettyPrintStruct(block)) } // Print out all balance changes in a given block. This does NOT exempt @@ -106,23 +105,58 @@ func runViewBlockCmd(cmd *cobra.Command, args []string) error { return fmt.Errorf("%w: unable to calculate balance changes", err) } + fmt.Printf("All Block Balance Changes (Hash: %s)\n", block.BlockIdentifier.Hash) + for _, balanceChange := range balanceChanges { parsedDiff, err := types.BigInt(balanceChange.Difference) if err != nil { return fmt.Errorf("%w: unable to parse Difference", err) } - log.Printf( - "%s -> %s\n", + fmt.Println( types.PrintStruct(balanceChange.Account), + "->", utils.PrettyAmount(parsedDiff, balanceChange.Currency), ) } + // Print out balance changes by transaction hash + // + // TODO: modify parser to allow for calculating balance + // changes for a single transaction. + for _, tx := range block.Transactions { + fmt.Printf("\n") + balanceChanges, err := p.BalanceChanges(Context, &types.Block{ + Transactions: []*types.Transaction{ + tx, + }, + }, false) + if err != nil { + return fmt.Errorf("%w: unable to calculate balance changes", err) + } + + fmt.Println("Transaction Hash:", tx.TransactionIdentifier.Hash) + + for _, balanceChange := range balanceChanges { + parsedDiff, err := types.BigInt(balanceChange.Difference) + if err != nil { + return fmt.Errorf("%w: unable to parse Difference", err) + } + + fmt.Println( + types.PrintStruct(balanceChange.Account), + "->", + utils.PrettyAmount(parsedDiff, balanceChange.Currency), + ) + } + + } + fmt.Printf("\n") + if !OnlyChanges { // Print out all OperationGroups for each transaction in a block. for _, tx := range block.Transactions { - log.Printf( + fmt.Printf( "Transaction %s Operation Groups: %s\n", tx.TransactionIdentifier.Hash, types.PrettyPrintStruct(parser.GroupOperations(tx)), From f38007bcfd91d39c586e15b3898fe0a1d1f445ee Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Thu, 22 Oct 2020 18:30:19 -0700 Subject: [PATCH 146/376] nits --- cmd/view_block.go | 1 - 1 file changed, 1 deletion(-) diff --git a/cmd/view_block.go b/cmd/view_block.go index 235f1a60..1bd727e4 100644 --- a/cmd/view_block.go +++ b/cmd/view_block.go @@ -149,7 +149,6 @@ func runViewBlockCmd(cmd *cobra.Command, args []string) error { utils.PrettyAmount(parsedDiff, balanceChange.Currency), ) } - } fmt.Printf("\n") From 91c65de5cd2bfe64c165265b4bb2fae8d9e3a06e Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Thu, 22 Oct 2020 18:42:37 -0700 Subject: [PATCH 147/376] Logging cleanup --- cmd/view_block.go | 63 +++++++++++++++++++++++++++-------------------- 1 file changed, 36 insertions(+), 27 deletions(-) diff --git a/cmd/view_block.go b/cmd/view_block.go index 1bd727e4..51262c9f 100644 --- a/cmd/view_block.go +++ b/cmd/view_block.go @@ -23,6 +23,7 @@ import ( "github.com/coinbase/rosetta-sdk-go/parser" "github.com/coinbase/rosetta-sdk-go/types" "github.com/coinbase/rosetta-sdk-go/utils" + "github.com/fatih/color" "github.com/spf13/cobra" ) @@ -44,6 +45,27 @@ fetch is formatted incorrectly.`, } ) +func printChanges(balanceChanges []*parser.BalanceChange) error { + for _, balanceChange := range balanceChanges { + parsedDiff, err := types.BigInt(balanceChange.Difference) + if err != nil { + return fmt.Errorf("%w: unable to parse Difference", err) + } + + if parsedDiff.Sign() == 0 { + continue + } + + fmt.Println( + types.PrintStruct(balanceChange.Account), + "->", + utils.PrettyAmount(parsedDiff, balanceChange.Currency), + ) + } + + return nil +} + func runViewBlockCmd(cmd *cobra.Command, args []string) error { index, err := strconv.ParseInt(args[0], 10, 64) if err != nil { @@ -93,39 +115,34 @@ func runViewBlockCmd(cmd *cobra.Command, args []string) error { return fmt.Errorf("%w: unable to fetch block", fetchErr.Err) } + fmt.Printf("\n") if !OnlyChanges { - fmt.Println("Current Block:", types.PrettyPrintStruct(block)) + color.Cyan("Current Block:") + fmt.Println(types.PrettyPrintStruct(block)) } // Print out all balance changes in a given block. This does NOT exempt // any operations/accounts from parsing. + color.Cyan("Balance Changes:") p := parser.New(newFetcher.Asserter, func(*types.Operation) bool { return false }, nil) balanceChanges, err := p.BalanceChanges(Context, block, false) if err != nil { return fmt.Errorf("%w: unable to calculate balance changes", err) } - fmt.Printf("All Block Balance Changes (Hash: %s)\n", block.BlockIdentifier.Hash) + fmt.Println("Cummulative:", block.BlockIdentifier.Hash) - for _, balanceChange := range balanceChanges { - parsedDiff, err := types.BigInt(balanceChange.Difference) - if err != nil { - return fmt.Errorf("%w: unable to parse Difference", err) - } - - fmt.Println( - types.PrintStruct(balanceChange.Account), - "->", - utils.PrettyAmount(parsedDiff, balanceChange.Currency), - ) + if err := printChanges(balanceChanges); err != nil { + return err } + fmt.Printf("\n") + // Print out balance changes by transaction hash // // TODO: modify parser to allow for calculating balance // changes for a single transaction. for _, tx := range block.Transactions { - fmt.Printf("\n") balanceChanges, err := p.BalanceChanges(Context, &types.Block{ Transactions: []*types.Transaction{ tx, @@ -135,25 +152,17 @@ func runViewBlockCmd(cmd *cobra.Command, args []string) error { return fmt.Errorf("%w: unable to calculate balance changes", err) } - fmt.Println("Transaction Hash:", tx.TransactionIdentifier.Hash) + fmt.Println("Transaction:", tx.TransactionIdentifier.Hash) - for _, balanceChange := range balanceChanges { - parsedDiff, err := types.BigInt(balanceChange.Difference) - if err != nil { - return fmt.Errorf("%w: unable to parse Difference", err) - } - - fmt.Println( - types.PrintStruct(balanceChange.Account), - "->", - utils.PrettyAmount(parsedDiff, balanceChange.Currency), - ) + if err := printChanges(balanceChanges); err != nil { + return err } + fmt.Printf("\n") } - fmt.Printf("\n") if !OnlyChanges { // Print out all OperationGroups for each transaction in a block. + color.Cyan("Operation Groups:") for _, tx := range block.Transactions { fmt.Printf( "Transaction %s Operation Groups: %s\n", From 739eb6d17e6322e4ecfeb319241013af9e80dd0e Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Thu, 22 Oct 2020 18:46:23 -0700 Subject: [PATCH 148/376] Cleanup README --- README.md | 43 +++++++++++++++++++++++++++++++++++++------ 1 file changed, 37 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 08e1329e..30f90e09 100644 --- a/README.md +++ b/README.md @@ -65,13 +65,16 @@ Available Commands: view:networks View all network statuses Flags: + --block-profile string Save the pprof block profile in the specified file --configuration-file string Configuration file that provides connection and test settings. If you would like to generate a starter configuration file (populated with the defaults), run rosetta-cli configuration:create. Any fields not populated in the configuration file will be populated with default values. + --cpu-profile string Save the pprof cpu profile in the specified file -h, --help help for rosetta-cli + --mem-profile string Save the pprof mem profile in the specified file Use "rosetta-cli [command] --help" for more information about a command. ``` @@ -206,6 +209,11 @@ by the `/account/balance` endpoint?). Take a look at the [simple configuration](examples/configuration/simple.json) for an example of how to do this. +#### Status Codes +If there are no issues found while running `check`, it will exit with a `0` status code. +If there are any issues, it will exit with a `1` status code. It can be useful +to run this command as an integration test for any changes to your implementation. + ### Commands #### version ``` @@ -218,12 +226,15 @@ Flags: -h, --help help for version Global Flags: + --block-profile string Save the pprof block profile in the specified file --configuration-file string Configuration file that provides connection and test settings. If you would like to generate a starter configuration file (populated with the defaults), run rosetta-cli configuration:create. Any fields not populated in the configuration file will be populated with default values. + --cpu-profile string Save the pprof cpu profile in the specified file + --mem-profile string Save the pprof mem profile in the specified file ``` #### check:data @@ -270,19 +281,17 @@ Flags: -h, --help help for check:data Global Flags: + --block-profile string Save the pprof block profile in the specified file --configuration-file string Configuration file that provides connection and test settings. If you would like to generate a starter configuration file (populated with the defaults), run rosetta-cli configuration:create. Any fields not populated in the configuration file will be populated with default values. + --cpu-profile string Save the pprof cpu profile in the specified file + --mem-profile string Save the pprof mem profile in the specified file ``` -##### Status Codes -If there are no issues found while running `check`, it will exit with a `0` status code. -If there are any issues, it will exit with a `1` status code. It can be useful -to run this command as an integration test for any changes to your implementation. - #### check:construction ``` The check:construction command runs an automated test of a @@ -309,12 +318,15 @@ Flags: -h, --help help for check:construction Global Flags: + --block-profile string Save the pprof block profile in the specified file --configuration-file string Configuration file that provides connection and test settings. If you would like to generate a starter configuration file (populated with the defaults), run rosetta-cli configuration:create. Any fields not populated in the configuration file will be populated with default values. + --cpu-profile string Save the pprof cpu profile in the specified file + --mem-profile string Save the pprof mem profile in the specified file ``` #### configuration:create @@ -328,12 +340,15 @@ Flags: -h, --help help for configuration:create Global Flags: + --block-profile string Save the pprof block profile in the specified file --configuration-file string Configuration file that provides connection and test settings. If you would like to generate a starter configuration file (populated with the defaults), run rosetta-cli configuration:create. Any fields not populated in the configuration file will be populated with default values. + --cpu-profile string Save the pprof cpu profile in the specified file + --mem-profile string Save the pprof mem profile in the specified file ``` #### configuration:validate @@ -371,12 +386,15 @@ Flags: -h, --help help for view:networks Global Flags: + --block-profile string Save the pprof block profile in the specified file --configuration-file string Configuration file that provides connection and test settings. If you would like to generate a starter configuration file (populated with the defaults), run rosetta-cli configuration:create. Any fields not populated in the configuration file will be populated with default values. + --cpu-profile string Save the pprof cpu profile in the specified file + --mem-profile string Save the pprof mem profile in the specified file ``` #### view:account @@ -397,12 +415,15 @@ Flags: -h, --help help for view:account Global Flags: + --block-profile string Save the pprof block profile in the specified file --configuration-file string Configuration file that provides connection and test settings. If you would like to generate a starter configuration file (populated with the defaults), run rosetta-cli configuration:create. Any fields not populated in the configuration file will be populated with default values. + --cpu-profile string Save the pprof cpu profile in the specified file + --mem-profile string Save the pprof mem profile in the specified file ``` #### view:block @@ -421,15 +442,19 @@ Usage: rosetta-cli view:block [flags] Flags: - -h, --help help for view:block + -h, --help help for view:block + --only-changes Only print balance changes for accounts in the block Global Flags: + --block-profile string Save the pprof block profile in the specified file --configuration-file string Configuration file that provides connection and test settings. If you would like to generate a starter configuration file (populated with the defaults), run rosetta-cli configuration:create. Any fields not populated in the configuration file will be populated with default values. + --cpu-profile string Save the pprof cpu profile in the specified file + --mem-profile string Save the pprof mem profile in the specified file ``` #### utils:asserter-configuration @@ -450,12 +475,15 @@ Flags: -h, --help help for utils:asserter-configuration Global Flags: + --block-profile string Save the pprof block profile in the specified file --configuration-file string Configuration file that provides connection and test settings. If you would like to generate a starter configuration file (populated with the defaults), run rosetta-cli configuration:create. Any fields not populated in the configuration file will be populated with default values. + --cpu-profile string Save the pprof cpu profile in the specified file + --mem-profile string Save the pprof mem profile in the specified file ``` #### utils:train-zstd @@ -479,12 +507,15 @@ Flags: -h, --help help for utils:train-zstd Global Flags: + --block-profile string Save the pprof block profile in the specified file --configuration-file string Configuration file that provides connection and test settings. If you would like to generate a starter configuration file (populated with the defaults), run rosetta-cli configuration:create. Any fields not populated in the configuration file will be populated with default values. + --cpu-profile string Save the pprof cpu profile in the specified file + --mem-profile string Save the pprof mem profile in the specified file ``` ## Correctness Checks From 6f39723f6dcdf102c3f0b7ca00a197ec6b92142a Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Sun, 25 Oct 2020 21:05:20 -0700 Subject: [PATCH 149/376] Update rosetta-sdk-go --- go.mod | 2 +- go.sum | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index 521935de..bdcd338f 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/coinbase/rosetta-cli go 1.13 require ( - github.com/coinbase/rosetta-sdk-go v0.5.6 + github.com/coinbase/rosetta-sdk-go v0.5.7-0.20201026040323-39bc21f235e7 github.com/fatih/color v1.9.0 github.com/olekukonko/tablewriter v0.0.2-0.20190409134802-7e037d187b0c github.com/spf13/cobra v1.1.1 diff --git a/go.sum b/go.sum index f68fd0ab..dc3763a3 100644 --- a/go.sum +++ b/go.sum @@ -82,6 +82,8 @@ github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDk github.com/cloudflare/cloudflare-go v0.10.2-0.20190916151808-a80f83b9add9/go.mod h1:1MxXX1Ux4x6mqPmjkUgTP1CdXIBXKX7T+Jk9Gxrmx+U= github.com/coinbase/rosetta-sdk-go v0.5.6 h1:mgSSw2NsVUBZPgeQegIWazJki0IT8Zd+xrFLS1afzJE= github.com/coinbase/rosetta-sdk-go v0.5.6/go.mod h1:JRO4BJjhWAI7nYGwzYZWFgYfCTsQOvdZB7tGyN6kEtE= +github.com/coinbase/rosetta-sdk-go v0.5.7-0.20201026040323-39bc21f235e7 h1:0QRe5bKAlLOKxO8ucH1o3fNvr24+Tems3ttTXk8srSM= +github.com/coinbase/rosetta-sdk-go v0.5.7-0.20201026040323-39bc21f235e7/go.mod h1:JRO4BJjhWAI7nYGwzYZWFgYfCTsQOvdZB7tGyN6kEtE= github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= From e1d9cac0704d5c845021ac52bab3fc7430619fd6 Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Sun, 25 Oct 2020 21:11:09 -0700 Subject: [PATCH 150/376] Add reconciler active backlog field to config --- configuration/types.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/configuration/types.go b/configuration/types.go index 2349f829..b05af71c 100644 --- a/configuration/types.go +++ b/configuration/types.go @@ -322,6 +322,11 @@ type DataConfiguration struct { // accounts are provided using bootstrap_balances and // syncing starts from genesis). InitialBalanceFetchDisabled bool `json:"initial_balance_fetch_disabled"` + + // ReconcilerActiveBacklog is the maximum number of pending changes + // to keep in the active reconciliation backlog before skipping + // reconciliation on new changes. + ReconcilerActiveBacklog *int `json:"reconciler_active_backlog"` } // Configuration contains all configuration settings for running From cfed67fd8848af7d4a9d33995d2ad864ecae33f9 Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Sun, 25 Oct 2020 21:31:34 -0700 Subject: [PATCH 151/376] Fix prune balances --- go.mod | 2 +- go.sum | 2 ++ pkg/processor/coordinator_helper.go | 15 +++++++++++---- pkg/processor/reconciler_helper.go | 20 +++++++++++++++++++- 4 files changed, 33 insertions(+), 6 deletions(-) diff --git a/go.mod b/go.mod index bdcd338f..b9fc867b 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/coinbase/rosetta-cli go 1.13 require ( - github.com/coinbase/rosetta-sdk-go v0.5.7-0.20201026040323-39bc21f235e7 + github.com/coinbase/rosetta-sdk-go v0.5.7-0.20201026043031-084899fafb19 github.com/fatih/color v1.9.0 github.com/olekukonko/tablewriter v0.0.2-0.20190409134802-7e037d187b0c github.com/spf13/cobra v1.1.1 diff --git a/go.sum b/go.sum index dc3763a3..1b32e8ec 100644 --- a/go.sum +++ b/go.sum @@ -84,6 +84,8 @@ github.com/coinbase/rosetta-sdk-go v0.5.6 h1:mgSSw2NsVUBZPgeQegIWazJki0IT8Zd+xrF github.com/coinbase/rosetta-sdk-go v0.5.6/go.mod h1:JRO4BJjhWAI7nYGwzYZWFgYfCTsQOvdZB7tGyN6kEtE= github.com/coinbase/rosetta-sdk-go v0.5.7-0.20201026040323-39bc21f235e7 h1:0QRe5bKAlLOKxO8ucH1o3fNvr24+Tems3ttTXk8srSM= github.com/coinbase/rosetta-sdk-go v0.5.7-0.20201026040323-39bc21f235e7/go.mod h1:JRO4BJjhWAI7nYGwzYZWFgYfCTsQOvdZB7tGyN6kEtE= +github.com/coinbase/rosetta-sdk-go v0.5.7-0.20201026043031-084899fafb19 h1:Z+eflZ5M9MuhYscTMXawDzh2To+RJVLjKjI7yGJA4LY= +github.com/coinbase/rosetta-sdk-go v0.5.7-0.20201026043031-084899fafb19/go.mod h1:JRO4BJjhWAI7nYGwzYZWFgYfCTsQOvdZB7tGyN6kEtE= github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= diff --git a/pkg/processor/coordinator_helper.go b/pkg/processor/coordinator_helper.go index 10dd75cc..16086fdd 100644 --- a/pkg/processor/coordinator_helper.go +++ b/pkg/processor/coordinator_helper.go @@ -16,6 +16,7 @@ package processor import ( "context" + "errors" "fmt" "log" "math/big" @@ -394,15 +395,21 @@ func (c *CoordinatorHelper) Balance( accountIdentifier *types.AccountIdentifier, currency *types.Currency, ) (*types.Amount, error) { - amount, _, err := c.balanceStorage.GetBalanceTransactional( + headBlock, err := c.blockStorage.GetHeadBlockIdentifier(ctx) + if err != nil { + return nil, err + } + if headBlock == nil { + return nil, errors.New("no blocks synced") + } + + return c.balanceStorage.GetBalanceTransactional( ctx, dbTx, accountIdentifier, currency, - nil, + headBlock, ) - - return amount, err } // Coins returns all *types.Coin owned by diff --git a/pkg/processor/reconciler_helper.go b/pkg/processor/reconciler_helper.go index 62d6fdbd..1b1defe4 100644 --- a/pkg/processor/reconciler_helper.go +++ b/pkg/processor/reconciler_helper.go @@ -79,7 +79,7 @@ func (h *ReconcilerHelper) ComputedBalance( account *types.AccountIdentifier, currency *types.Currency, headBlock *types.BlockIdentifier, -) (*types.Amount, *types.BlockIdentifier, error) { +) (*types.Amount, error) { return h.balanceStorage.GetBalance(ctx, account, currency, headBlock) } @@ -103,3 +103,21 @@ func (h *ReconcilerHelper) LiveBalance( } return amt, block, nil } + +// PruneBalances removes all historical balance states +// <= some index. This can significantly reduce storage +// usage in scenarios where historical balances are only +// retrieved once (like reconciliation). +func (h *ReconcilerHelper) PruneBalances( + ctx context.Context, + account *types.AccountIdentifier, + currency *types.Currency, + index int64, +) error { + return h.balanceStorage.PruneBalances( + ctx, + account, + currency, + index, + ) +} From 8672a226beb58f680062ae926f7c71b6e3cc121e Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Sun, 25 Oct 2020 21:35:41 -0700 Subject: [PATCH 152/376] Populate custom backlog --- pkg/tester/data.go | 25 ++++++++++++++++++------- 1 file changed, 18 insertions(+), 7 deletions(-) diff --git a/pkg/tester/data.go b/pkg/tester/data.go index c4e52215..e0087535 100644 --- a/pkg/tester/data.go +++ b/pkg/tester/data.go @@ -248,17 +248,28 @@ func InitializeData( historicalBalanceEnabled = networkOptions.Allow.HistoricalBalanceLookup } - r := reconciler.New( - reconcilerHelper, - reconcilerHandler, - parser, + rOpts := []reconciler.Option{ reconciler.WithActiveConcurrency(int(config.Data.ActiveReconciliationConcurrency)), reconciler.WithInactiveConcurrency(int(config.Data.InactiveReconciliationConcurrency)), - reconciler.WithLookupBalanceByBlock(historicalBalanceEnabled), reconciler.WithInterestingAccounts(interestingAccounts), reconciler.WithSeenAccounts(seenAccounts), - reconciler.WithDebugLogging(config.Data.LogReconciliations), reconciler.WithInactiveFrequency(int64(config.Data.InactiveReconciliationFrequency)), + } + if config.Data.ReconcilerActiveBacklog != nil { + rOpts = append(rOpts, reconciler.WithBacklogSize(*config.Data.ReconcilerActiveBacklog)) + } + if historicalBalanceEnabled { + rOpts = append(rOpts, reconciler.WithLookupBalanceByBlock()) + } + if config.Data.LogReconciliations { + rOpts = append(rOpts, reconciler.WithDebugLogging()) + } + + r := reconciler.New( + reconcilerHelper, + reconcilerHandler, + parser, + rOpts..., ) blockWorkers := []storage.BlockWorker{} @@ -960,7 +971,7 @@ func (t *DataTester) recursiveOpSearch( // Do not do any inactive lookups when looking for the block with missing // operations. reconciler.WithInactiveConcurrency(0), - reconciler.WithLookupBalanceByBlock(t.historicalBalanceEnabled), + reconciler.WithLookupBalanceByBlock(), reconciler.WithInterestingAccounts([]*reconciler.AccountCurrency{accountCurrency}), ) From c98bdaeda66494325304f5071ae08710a48cd241 Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Sun, 25 Oct 2020 21:41:40 -0700 Subject: [PATCH 153/376] Fix genesis add issue --- go.mod | 2 +- go.sum | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index b9fc867b..1c830606 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/coinbase/rosetta-cli go 1.13 require ( - github.com/coinbase/rosetta-sdk-go v0.5.7-0.20201026043031-084899fafb19 + github.com/coinbase/rosetta-sdk-go v0.5.7-0.20201026044055-a989b53e154d github.com/fatih/color v1.9.0 github.com/olekukonko/tablewriter v0.0.2-0.20190409134802-7e037d187b0c github.com/spf13/cobra v1.1.1 diff --git a/go.sum b/go.sum index 1b32e8ec..4872eca0 100644 --- a/go.sum +++ b/go.sum @@ -86,6 +86,8 @@ github.com/coinbase/rosetta-sdk-go v0.5.7-0.20201026040323-39bc21f235e7 h1:0QRe5 github.com/coinbase/rosetta-sdk-go v0.5.7-0.20201026040323-39bc21f235e7/go.mod h1:JRO4BJjhWAI7nYGwzYZWFgYfCTsQOvdZB7tGyN6kEtE= github.com/coinbase/rosetta-sdk-go v0.5.7-0.20201026043031-084899fafb19 h1:Z+eflZ5M9MuhYscTMXawDzh2To+RJVLjKjI7yGJA4LY= github.com/coinbase/rosetta-sdk-go v0.5.7-0.20201026043031-084899fafb19/go.mod h1:JRO4BJjhWAI7nYGwzYZWFgYfCTsQOvdZB7tGyN6kEtE= +github.com/coinbase/rosetta-sdk-go v0.5.7-0.20201026044055-a989b53e154d h1:A0XicDOrvyhlKBKl0qT1QqHZv3VfUurQSqHsWJUxjyk= +github.com/coinbase/rosetta-sdk-go v0.5.7-0.20201026044055-a989b53e154d/go.mod h1:JRO4BJjhWAI7nYGwzYZWFgYfCTsQOvdZB7tGyN6kEtE= github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= From df67e3049890b2de768e9819a90a6523c7a8e52e Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Mon, 26 Oct 2020 08:12:49 -0700 Subject: [PATCH 154/376] Add balance pruning --- pkg/tester/data.go | 1 + 1 file changed, 1 insertion(+) diff --git a/pkg/tester/data.go b/pkg/tester/data.go index e0087535..23ff0892 100644 --- a/pkg/tester/data.go +++ b/pkg/tester/data.go @@ -254,6 +254,7 @@ func InitializeData( reconciler.WithInterestingAccounts(interestingAccounts), reconciler.WithSeenAccounts(seenAccounts), reconciler.WithInactiveFrequency(int64(config.Data.InactiveReconciliationFrequency)), + reconciler.WithBalancePruning(), } if config.Data.ReconcilerActiveBacklog != nil { rOpts = append(rOpts, reconciler.WithBacklogSize(*config.Data.ReconcilerActiveBacklog)) From c3359c7cb9d5425d327a2941de559a240287cd6b Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Mon, 26 Oct 2020 09:50:31 -0700 Subject: [PATCH 155/376] Update to v0.5.7 --- go.mod | 2 +- go.sum | 92 ++++------------------------------------------------------ 2 files changed, 7 insertions(+), 87 deletions(-) diff --git a/go.mod b/go.mod index 1c830606..cd24ad04 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/coinbase/rosetta-cli go 1.13 require ( - github.com/coinbase/rosetta-sdk-go v0.5.7-0.20201026044055-a989b53e154d + github.com/coinbase/rosetta-sdk-go v0.5.7 github.com/fatih/color v1.9.0 github.com/olekukonko/tablewriter v0.0.2-0.20190409134802-7e037d187b0c github.com/spf13/cobra v1.1.1 diff --git a/go.sum b/go.sum index 4872eca0..f68f7846 100644 --- a/go.sum +++ b/go.sum @@ -38,11 +38,7 @@ github.com/VictoriaMetrics/fastcache v1.5.7 h1:4y6y0G8PRzszQUYIQHHssv/jgPHAb5qQu github.com/VictoriaMetrics/fastcache v1.5.7/go.mod h1:ptDBkNMQI4RtmVo8VS/XwRY6RoTu1dAWCbrk+6WsEM8= github.com/aead/siphash v1.0.1/go.mod h1:Nywa3cDsYNNK3gaciGTWPwHt0wlpNV15vwmswBAUSII= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= -github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 h1:JYp7IbQjafoB+tBA3gMyHYHrpOtNuDiK/uB5uXxq5wM= -github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= -github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d h1:UQZhZ2O0vMHr2cI+DC1Mbh0TJxzA3RcLoMsFw+aXw7E= -github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho= github.com/allegro/bigcache v1.2.1-0.20190218064605-e24eb225f156/go.mod h1:Cb/ax3seSYIx7SuZdm2G2xzfwmv3TPSk2ucNfQESPXM= github.com/aristanetworks/goarista v0.0.0-20170210015632-ea17b1a17847 h1:rtI0fD4oG/8eVokGVPYJEW1F88p1ZNgXiEIs9thEE4A= github.com/aristanetworks/goarista v0.0.0-20170210015632-ea17b1a17847/go.mod h1:D/tb0zPVXnP7fmsLZjtdUhSsumbK/ij54UXjjVgMGxQ= @@ -80,14 +76,8 @@ github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+ github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cloudflare/cloudflare-go v0.10.2-0.20190916151808-a80f83b9add9/go.mod h1:1MxXX1Ux4x6mqPmjkUgTP1CdXIBXKX7T+Jk9Gxrmx+U= -github.com/coinbase/rosetta-sdk-go v0.5.6 h1:mgSSw2NsVUBZPgeQegIWazJki0IT8Zd+xrFLS1afzJE= -github.com/coinbase/rosetta-sdk-go v0.5.6/go.mod h1:JRO4BJjhWAI7nYGwzYZWFgYfCTsQOvdZB7tGyN6kEtE= -github.com/coinbase/rosetta-sdk-go v0.5.7-0.20201026040323-39bc21f235e7 h1:0QRe5bKAlLOKxO8ucH1o3fNvr24+Tems3ttTXk8srSM= -github.com/coinbase/rosetta-sdk-go v0.5.7-0.20201026040323-39bc21f235e7/go.mod h1:JRO4BJjhWAI7nYGwzYZWFgYfCTsQOvdZB7tGyN6kEtE= -github.com/coinbase/rosetta-sdk-go v0.5.7-0.20201026043031-084899fafb19 h1:Z+eflZ5M9MuhYscTMXawDzh2To+RJVLjKjI7yGJA4LY= -github.com/coinbase/rosetta-sdk-go v0.5.7-0.20201026043031-084899fafb19/go.mod h1:JRO4BJjhWAI7nYGwzYZWFgYfCTsQOvdZB7tGyN6kEtE= -github.com/coinbase/rosetta-sdk-go v0.5.7-0.20201026044055-a989b53e154d h1:A0XicDOrvyhlKBKl0qT1QqHZv3VfUurQSqHsWJUxjyk= -github.com/coinbase/rosetta-sdk-go v0.5.7-0.20201026044055-a989b53e154d/go.mod h1:JRO4BJjhWAI7nYGwzYZWFgYfCTsQOvdZB7tGyN6kEtE= +github.com/coinbase/rosetta-sdk-go v0.5.7 h1:BaR/+O3GzrsyunVNkVQHtjDCcId8G1Fh/RqEbeyExnk= +github.com/coinbase/rosetta-sdk-go v0.5.7/go.mod h1:l5aNeyeZKBkmWbVdkdLpWdToQ6hTwI7cZ1OU9cMbljY= github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= @@ -99,12 +89,6 @@ github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfc github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE= github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= -github.com/dave/dst v0.23.1 h1:2obX6c3RqALrEOp6u01qsqPvwp0t+RpOp9O4Bf9KhXs= -github.com/dave/dst v0.23.1/go.mod h1:LjPcLEauK4jC5hQ1fE/wr05O41zK91Pr4Qs22Ljq7gs= -github.com/dave/gopackages v0.0.0-20170318123100-46e7023ec56e/go.mod h1:i00+b/gKdIDIxuLDFob7ustLAVqhsZRk2qVZrArELGQ= -github.com/dave/jennifer v1.2.0/go.mod h1:fIb+770HOpJ2fmN9EPPKOqm1vMGhB+TwXKMZhrIygKg= -github.com/dave/kerr v0.0.0-20170318121727-bc25dd6abe8e/go.mod h1:qZqlPyPvfsDJt+3wHJ1EvSXDuVjFTK0j2p/ca+gtsb8= -github.com/dave/rebecca v0.9.1/go.mod h1:N6XYdMD/OKw3lkF3ywh8Z6wPGuwNFDNtWYEMFWEmXBA= github.com/davecgh/go-spew v0.0.0-20171005155431-ecdeabc65495/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= @@ -137,8 +121,6 @@ github.com/fatih/color v1.3.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5Kwzbycv github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/fatih/color v1.9.0 h1:8xPHl4/q1VyqGIPif1F+1V3Y3lSmrq01EabUW3CoW5s= github.com/fatih/color v1.9.0/go.mod h1:eQcE1qtQxscV5RaZvpXrrb8Drkc3/DdQ+uUYCNjL+zU= -github.com/fatih/structtag v1.2.0 h1:/OdNE99OxoI/PqaW/SuSK9uxxT3f/tcSZgon/ssNSx4= -github.com/fatih/structtag v1.2.0/go.mod h1:mBJUNpUnHmRKrKlQQlmCrh5PuhftFbNv8Ys4/aAZl94= github.com/fjl/memsize v0.0.0-20180418122429-ca190fb6ffbc/go.mod h1:VvhXpOYNQvB+uIk2RvXzuaQtkQJzzIx6lSBe1xv7hi0= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= @@ -164,7 +146,6 @@ github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5y github.com/golang/protobuf v1.3.1 h1:YF8+flBXS5eO826T4nzqPrxfhQThhXl0YzfuUPu4SBg= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.3.4/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= @@ -176,8 +157,6 @@ github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.2-0.20200707131729-196ae77b8a26 h1:lMm2hD9Fy0ynom5+85/pbdkiYcBqM1JWmhpAXLmy0fw= github.com/golang/snappy v0.0.2-0.20200707131729-196ae77b8a26/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= -github.com/google/addlicense v0.0.0-20200827091314-d1655b921368 h1:Ds6gDZHoviaQM7r7oMx/cG2qwZc3l5u7cg6gTkxOZNE= -github.com/google/addlicense v0.0.0-20200827091314-d1655b921368/go.mod h1:EMjYTRimagHs1FwlIqKyX3wAM0u3rA+McvlIIWmSamA= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= @@ -187,12 +166,10 @@ github.com/google/go-cmp v0.4.0 h1:xsAVV57WRhGj6kEIi8ReJzQlHHqcBYCElAvkovg3B/4= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.0 h1:/QaMHBdZ26BB3SSst0Iwl10Epc+xhTquomWX0oZEB6w= github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.2 h1:X2ev0eStA3AbceY54o37/0PQ/UWqKEiiO2dKL5OPaFM= github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/gofuzz v1.1.1-0.20200604201612-c04b05f3adfa/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= -github.com/google/pprof v0.0.0-20181127221834-b4f47329b966/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= @@ -231,7 +208,6 @@ github.com/holiman/uint256 v1.1.1/go.mod h1:y4ga/t+u+Xwd7CpDgZESaRcWy0I7XMlTMA25 github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/huin/goupnp v1.0.0/go.mod h1:n9v9KO1tAxYH82qOn+UTIFQDmx5n1Zxd/ClZDMX7Bnc= github.com/huin/goutil v0.0.0-20170803182201-1ca381bf3150/go.mod h1:PpLOETDnJ0o3iZrZfqZzyLl6l7F3c6L1oWn7OICBi6o= -github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/influxdata/influxdb v1.2.3-0.20180221223340-01288bdb0883/go.mod h1:qZna6X/4elxqT3yI9iZYdZrWWdeFOOprn86kgg4+IzY= @@ -251,7 +227,6 @@ github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvW github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/kkdai/bstream v0.0.0-20161212061736-f391b8402d23/go.mod h1:J+Gs4SYgM6CZQHDETBtE9HaSEkGmuNXF86RwHhHUvq4= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= -github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.2.0 h1:s5hAObm+yFO5uHYt5dYjxi2rXrsnmRpJx4OYvIWUaQs= @@ -275,7 +250,6 @@ github.com/mattn/go-ieproxy v0.0.0-20190702010315-6dee0af9227d/go.mod h1:31jz6HN github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-isatty v0.0.5-0.20180830101745-3fb116b82035/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= -github.com/mattn/go-isatty v0.0.10/go.mod h1:qgIWMr58cqv1PHHyhnkY9lrL7etaEgOFcMEpPG5Rm84= github.com/mattn/go-isatty v0.0.11 h1:FxPOTFNqGkuDUGi3H/qkUbQO4ZiBa2brKq5r0l8TGeM= github.com/mattn/go-isatty v0.0.11/go.mod h1:PhnuNfih5lzO57/f3n+odYbM4JtupLOxQOAqxQCu2WE= github.com/mattn/go-isatty v0.0.12 h1:wuysRhFDzyxgEmMf5xjvJ2M9dZoWAXNNr5LSBS7uHXY= @@ -284,8 +258,6 @@ github.com/mattn/go-runewidth v0.0.3/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzp github.com/mattn/go-runewidth v0.0.4 h1:2BvfKmzob6Bmd4YsL0zygOqfdFnK7GR4QL06Do4/p7Y= github.com/mattn/go-runewidth v0.0.4/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= -github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b h1:j7+1HpAFS1zy5+Q4qx1fWh90gTKwiN4QCGoY9TWyyO4= -github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b/go.mod h1:01TrycV0kFyexm33Z7vhZRXopbI8J3TDReVlkTgMUxE= github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc= github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= @@ -309,12 +281,10 @@ github.com/olekukonko/tablewriter v0.0.2-0.20190409134802-7e037d187b0c h1:1RHs3t github.com/olekukonko/tablewriter v0.0.2-0.20190409134802-7e037d187b0c/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= -github.com/onsi/ginkgo v1.10.2/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= github.com/onsi/ginkgo v1.14.0/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY= github.com/onsi/gomega v1.4.1/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= -github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= @@ -344,25 +314,16 @@ github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40T github.com/rjeczalik/notify v0.9.1/go.mod h1:rKwnCoCGeuQnwBtTSPL9Dad03Vh2n40ePRrjvIXnJho= github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= -github.com/rogpeppe/go-internal v1.5.2/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= -github.com/rogpeppe/go-internal v1.6.0/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= -github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= github.com/rs/cors v0.0.0-20160617231935-a62a804a8a00/go.mod h1:gFx+x8UowdsKA9AchylcLynDq+nNFfI8FkUZdN/jGCU= github.com/rs/xhandler v0.0.0-20160618193221-ed27b6fd6521/go.mod h1:RvLn4FgxWubrpZHtQLnOf6EwhN2hEMusxZOhcW9H3UQ= github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= -github.com/segmentio/golines v0.0.0-20200824192126-7f30d3046793 h1:rhR7esJSmty+9ST6Gsp7mlQHkpISw2DiYjuFaz3dRDg= -github.com/segmentio/golines v0.0.0-20200824192126-7f30d3046793/go.mod h1:bQSh5qdVR67XiCKbaVvYO41s50c5hQo+3cY/1CQQ3xQ= -github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= -github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= github.com/shirou/gopsutil v2.20.5+incompatible h1:tYH07UPoQt0OCQdgWWMgYHy3/a9bcxNpBIysykNIP7I= github.com/shirou/gopsutil v2.20.5+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA= github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= -github.com/sirupsen/logrus v1.4.2 h1:SPIRibHv4MatM3XXNO2BJeFLZwZ2LvZgfQ5+UNI2im4= -github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= @@ -394,8 +355,6 @@ github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+ github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= -github.com/stretchr/testify v1.5.1 h1:nOGnQDM7FYENwehXlg/kFVnos3rEvtKTjRvOWSzb6H4= -github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= @@ -412,30 +371,22 @@ github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1 github.com/tyler-smith/go-bip39 v1.0.1-0.20181017060643-dbb3b84ba2ef/go.mod h1:sJ5fKU0s6JVwZjjcUEX2zFOnvq0ASQ2K9Zr6cf67kNs= github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= -github.com/vmihailenco/msgpack/v4 v4.3.11/go.mod h1:gborTTJjAo/GWTqqRjrLCn9pgNN+NXzzngzBKDPIqw4= -github.com/vmihailenco/msgpack/v5 v5.0.0-beta.1 h1:d71/KA0LhvkrJ/Ok+Wx9qK7bU8meKA1Hk0jpVI5kJjk= -github.com/vmihailenco/msgpack/v5 v5.0.0-beta.1/go.mod h1:xlngVLeyQ/Qi05oQxhQ+oTuqa03RjMwMfk/7/TCs+QI= -github.com/vmihailenco/tagparser v0.1.1 h1:quXMXlA39OCbd2wAdTsGDlK9RkOk6Wuw+x37wVyIuWY= -github.com/vmihailenco/tagparser v0.1.1/go.mod h1:OeAg3pn3UbLjkWt+rN9oFYB6u/cQgqMEUPoW2WPyhdI= +github.com/vmihailenco/msgpack/v5 v5.0.0-beta.8 h1:R2L6zPq1pWFumpeIxAJoeiov5GxyEZUq9NyS8eus/6s= +github.com/vmihailenco/msgpack/v5 v5.0.0-beta.8/go.mod h1:HVxBVPUK/+fZMonk4bi1islLa8V3cfnBug0+4dykPzo= +github.com/vmihailenco/tagparser v0.1.2 h1:gnjoVuB/kljJ5wICEEOpx98oXMWPLj22G67Vbd1qPqc= +github.com/vmihailenco/tagparser v0.1.2/go.mod h1:OeAg3pn3UbLjkWt+rN9oFYB6u/cQgqMEUPoW2WPyhdI= github.com/wsddn/go-ecdh v0.0.0-20161211032359-48726bab9208/go.mod h1:IotVbo4F+mw0EzQ08zFqg7pK3FebNXpaMsRy2RT+Ees= -github.com/x-cray/logrus-prefixed-formatter v0.5.2 h1:00txxvfBM9muc0jiLIEAkAcIMJzfthRT6usrui8uGmg= -github.com/x-cray/logrus-prefixed-formatter v0.5.2/go.mod h1:2duySbKsL6M18s5GU7VPsoEPHyzalCE06qoARUCeBBE= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= -github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= -golang.org/x/arch v0.0.0-20180920145803-b19384d3c130/go.mod h1:cYlCBUl1MsqxdiKgmc4uh7TxZfWSFLOGSRR090WDxt8= golang.org/x/crypto v0.0.0-20170930174604-9419663f5a44/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= -golang.org/x/crypto v0.0.0-20181127143415-eb0de9b17e85/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= @@ -471,9 +422,6 @@ golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKG golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= golang.org/x/mod v0.1.1-0.20191209134235-331c550502dd/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.3.0 h1:RM4zey1++hCTbCVQfnWeKs9/IEsaBLA8vTkd0WVtmH4= -golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/net v0.0.0-20180719180050-a680a1efc54d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -492,11 +440,7 @@ golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= golang.org/x/net v0.0.0-20190620200207-3b0461eec859 h1:R/3boaszxrf1GEUWTVDzSKVwLmSJpwZ1yqXm8j0v2QI= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200625001655-4c5254603344 h1:vGXIOMxbNfDTk/aXCmfdLgkrSV+Z2tcbze+pEc3v5W4= -golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20200813134508-3edf25e44fcc h1:zK/HqS5bZxDptfPJNq8v7vJfXtkU7r9TLIoSr1bXaP4= golang.org/x/net v0.0.0-20200813134508-3edf25e44fcc/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= @@ -510,12 +454,10 @@ golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208 h1:qwRHBd0NqMbJxfbotnDhm2ByMI1Shq4Y6oRJo21SGJA= golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20180903190138-2b024373dcd9/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181026203630-95b1ffbd15a5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -526,7 +468,6 @@ golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5h golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -535,9 +476,6 @@ golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb h1:fgwFCsaw9buMuxNd6+DQfAuSF golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191008105621-543471e840be/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191024172528-b4ff53e7a1cb h1:ZxSglHghKPYD8WDeRUzRJrUJtDF0PxsTUSxyqr9/5BI= -golang.org/x/sys v0.0.0-20191024172528-b4ff53e7a1cb/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -556,7 +494,6 @@ golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxb golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20181127232545-e782529d0ddd/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= @@ -572,18 +509,9 @@ golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgw golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191024220359-3d91e92cde03 h1:4gtJXHJ9ud0q8MNSDxJsRU/WH+afypbe4Vk4zq+8qow= -golang.org/x/tools v0.0.0-20191024220359-3d91e92cde03/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191112195655-aa38f8e97acc/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191130070609-6e064ea0cf2d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20200117012304-6edc0a871e69/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200626171337-aa94e735be7f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20200729181040-64cdafbe085c/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= -golang.org/x/tools v0.0.0-20200731060945-b5fad4ed8dd6/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= -golang.org/x/tools v0.0.0-20200904185747-39188db58858 h1:xLt+iB5ksWcZVxqc+g9K41ZHy+6MKWfXCDsjSThnsPA= -golang.org/x/tools v0.0.0-20200904185747-39188db58858/go.mod h1:Cj7w3i3Rnn0Xh82ur9kSqwfTHTeVxaDqrfMjpcNT6bE= -golang.org/x/tools/gopls v0.4.4/go.mod h1:zhyGzA+CAtREUwwq/btQxEx2FHnGzDwJvGs5YqdVCbE= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= @@ -599,8 +527,6 @@ google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9Ywl google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= -google.golang.org/appengine v1.6.5 h1:tycE03LOZYQNhDpS27tcQdAzLCVMaj7QT2SXxebnpCM= -google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= @@ -641,7 +567,6 @@ gopkg.in/ini.v1 v1.51.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/natefinch/npipe.v2 v2.0.0-20160621034901-c1b8fa8bdcce/go.mod h1:5AcXVHNjg+BDxry382+8OKon8SEWiKktQR07RKPsv1c= gopkg.in/olebedev/go-duktape.v3 v3.0.0-20200619000410-60c24ae608a6/go.mod h1:uAJfkITjFhyEEuUfm7bsmCZRbW5WRq8s9EY8HZ6hCns= gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= -gopkg.in/src-d/go-billy.v4 v4.3.0/go.mod h1:tm33zBoOwxjYHZIE+OV8bxTWFMJLrconzFMd38aARFk= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= gopkg.in/urfave/cli.v1 v1.20.0/go.mod h1:vuBzUtMdQeixQj8LVd+/98pzhxNGQoyuPBlsXHOQNO0= gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= @@ -659,9 +584,4 @@ honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWh honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= -honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= -honnef.co/go/tools v0.0.1-2020.1.5/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= -mvdan.cc/gofumpt v0.0.0-20200709182408-4fd085cb6d5f/go.mod h1:9VQ397fNXEnF84t90W4r4TRCQK+pg9f8ugVfyj+S26w= -mvdan.cc/gofumpt v0.0.0-20200802201014-ab5a8192947d/go.mod h1:bzrjFmaD6+xqohD3KYP0H2FEuxknnBmyyOxdhLdaIws= -mvdan.cc/xurls/v2 v2.2.0/go.mod h1:EV1RMtya9D6G5DMYPGD8zTQzaHet6Jh8gFlRgGRJeO8= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= From 16cd8d4d598306c5079278657dbed37a8db9e2c3 Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Mon, 26 Oct 2020 09:54:59 -0700 Subject: [PATCH 156/376] nits --- configuration/types.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configuration/types.go b/configuration/types.go index b05af71c..9a6a51d8 100644 --- a/configuration/types.go +++ b/configuration/types.go @@ -326,7 +326,7 @@ type DataConfiguration struct { // ReconcilerActiveBacklog is the maximum number of pending changes // to keep in the active reconciliation backlog before skipping // reconciliation on new changes. - ReconcilerActiveBacklog *int `json:"reconciler_active_backlog"` + ReconcilerActiveBacklog *int `json:"reconciler_active_backlog,omitempty"` } // Configuration contains all configuration settings for running From 1c4869c7eb5ca2302a700e579bb708b4fbf043b6 Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Tue, 27 Oct 2020 15:28:13 -0700 Subject: [PATCH 157/376] Update rosetta-sdk-go --- go.mod | 2 +- go.sum | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index cd24ad04..2ded3f9c 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/coinbase/rosetta-cli go 1.13 require ( - github.com/coinbase/rosetta-sdk-go v0.5.7 + github.com/coinbase/rosetta-sdk-go v0.5.8-0.20201027222031-dd9e29377d5f github.com/fatih/color v1.9.0 github.com/olekukonko/tablewriter v0.0.2-0.20190409134802-7e037d187b0c github.com/spf13/cobra v1.1.1 diff --git a/go.sum b/go.sum index f68f7846..47fae483 100644 --- a/go.sum +++ b/go.sum @@ -78,6 +78,8 @@ github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDk github.com/cloudflare/cloudflare-go v0.10.2-0.20190916151808-a80f83b9add9/go.mod h1:1MxXX1Ux4x6mqPmjkUgTP1CdXIBXKX7T+Jk9Gxrmx+U= github.com/coinbase/rosetta-sdk-go v0.5.7 h1:BaR/+O3GzrsyunVNkVQHtjDCcId8G1Fh/RqEbeyExnk= github.com/coinbase/rosetta-sdk-go v0.5.7/go.mod h1:l5aNeyeZKBkmWbVdkdLpWdToQ6hTwI7cZ1OU9cMbljY= +github.com/coinbase/rosetta-sdk-go v0.5.8-0.20201027222031-dd9e29377d5f h1:aWkN9dKMkMMpZKX5QycpePxH176Fj2fNNC7jESfLZw0= +github.com/coinbase/rosetta-sdk-go v0.5.8-0.20201027222031-dd9e29377d5f/go.mod h1:l5aNeyeZKBkmWbVdkdLpWdToQ6hTwI7cZ1OU9cMbljY= github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= From 20e0ecfd00b05252c6a5f0cfb488468f5089e0c6 Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Tue, 27 Oct 2020 15:28:31 -0700 Subject: [PATCH 158/376] Address breaking changes --- pkg/processor/coordinator_helper.go | 2 +- pkg/processor/reconciler_helper.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/processor/coordinator_helper.go b/pkg/processor/coordinator_helper.go index 16086fdd..f21b8a86 100644 --- a/pkg/processor/coordinator_helper.go +++ b/pkg/processor/coordinator_helper.go @@ -403,7 +403,7 @@ func (c *CoordinatorHelper) Balance( return nil, errors.New("no blocks synced") } - return c.balanceStorage.GetBalanceTransactional( + return c.balanceStorage.GetOrSetBalanceTransactional( ctx, dbTx, accountIdentifier, diff --git a/pkg/processor/reconciler_helper.go b/pkg/processor/reconciler_helper.go index 1b1defe4..bd1bcf77 100644 --- a/pkg/processor/reconciler_helper.go +++ b/pkg/processor/reconciler_helper.go @@ -80,7 +80,7 @@ func (h *ReconcilerHelper) ComputedBalance( currency *types.Currency, headBlock *types.BlockIdentifier, ) (*types.Amount, error) { - return h.balanceStorage.GetBalance(ctx, account, currency, headBlock) + return h.balanceStorage.GetOrSetBalance(ctx, account, currency, headBlock) } // LiveBalance returns the live balance of an account. From 0f06ed7972d0552150cc9092bff83dcf54bfe6bd Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Tue, 27 Oct 2020 15:51:51 -0700 Subject: [PATCH 159/376] Remove unmaintained examples --- Makefile | 2 - examples/configuration/bitcoin.json | 383 --------------------------- examples/configuration/ethereum.json | 55 ---- examples/configuration/ethereum.ros | 138 ---------- 4 files changed, 578 deletions(-) delete mode 100644 examples/configuration/bitcoin.json delete mode 100644 examples/configuration/ethereum.json delete mode 100644 examples/configuration/ethereum.ros diff --git a/Makefile b/Makefile index 9a5416cc..e716ddc7 100644 --- a/Makefile +++ b/Makefile @@ -30,8 +30,6 @@ check-format: ! gofmt -s -l . | read; validate-configuration-files: - go run main.go configuration:validate examples/configuration/bitcoin.json; - go run main.go configuration:validate examples/configuration/ethereum.json; go run main.go configuration:validate examples/configuration/simple.json; go run main.go configuration:create examples/configuration/default.json; go run main.go configuration:validate examples/configuration/default.json; diff --git a/examples/configuration/bitcoin.json b/examples/configuration/bitcoin.json deleted file mode 100644 index 9afa31d5..00000000 --- a/examples/configuration/bitcoin.json +++ /dev/null @@ -1,383 +0,0 @@ -{ - "network": { - "blockchain": "Bitcoin", - "network": "Testnet3" - }, - "data_directory": "bitcoin-data", - "http_timeout": 300, - "max_retries": 5, - "retry_elapsed_time": 0, - "max_online_connections": 0, - "max_sync_concurrency": 0, - "tip_delay": 1800, - "log_configuration": false, - "construction": { - "max_offline_connections": 0, - "stale_depth": 0, - "broadcast_limit": 0, - "ignore_broadcast_failures": false, - "clear_broadcasts": false, - "broadcast_behind_tip": false, - "block_broadcast_limit": 0, - "rebroadcast_all": false, - "workflows": [ - { - "name": "request_funds", - "concurrency": 1, - "scenarios": [ - { - "name": "find_account", - "actions": [ - { - "input": "{\"symbol\":\"tBTC\", \"decimals\":8}", - "type": "set_variable", - "output_path": "currency" - }, - { - "input": "{\"minimum_balance\":{\"value\": \"0\", \"currency\": {{currency}}}, \"create_limit\":1}", - "type": "find_balance", - "output_path": "random_account" - } - ] - }, - { - "name": "request", - "actions": [ - { - "input": "{\"account_identifier\": {{random_account.account_identifier}}, \"minimum_balance\":{\"value\": \"1000000\", \"currency\": {{currency}}}}", - "type": "find_balance", - "output_path": "loaded_account" - } - ] - } - ] - }, - { - "name": "create_account", - "concurrency": 1, - "scenarios": [ - { - "name": "create_account", - "actions": [ - { - "input": "{\"network\":\"Testnet3\", \"blockchain\":\"Bitcoin\"}", - "type": "set_variable", - "output_path": "network" - }, - { - "input": "{\"curve_type\": \"secp256k1\"}", - "type": "generate_key", - "output_path": "key" - }, - { - "input": "{\"network_identifier\": {{network}}, \"public_key\": {{key.public_key}}}", - "type": "derive", - "output_path": "account" - }, - { - "input": "{\"account_identifier\": {{account.account_identifier}}, \"keypair\": {{key}}}", - "type": "save_account" - } - ] - } - ] - }, - { - "name": "transfer", - "concurrency": 10, - "scenarios": [ - { - "name": "transfer_dry_run", - "actions": [ - { - "input": "{\"network\":\"Testnet3\", \"blockchain\":\"Bitcoin\"}", - "type": "set_variable", - "output_path": "transfer_dry_run.network" - }, - { - "input": "{\"symbol\":\"tBTC\", \"decimals\":8}", - "type": "set_variable", - "output_path": "currency" - }, - { - "input": "\"600\"", - "type": "set_variable", - "output_path": "dust_amount" - }, - { - "input": "\"1200\"", - "type": "set_variable", - "output_path": "max_fee_amount" - }, - { - "input": "{\"operation\":\"addition\", \"left_value\": {{dust_amount}}, \"right_value\": {{max_fee_amount}}}", - "type": "math", - "output_path": "send_buffer" - }, - { - "input": "\"2400\"", - "type": "set_variable", - "output_path": "reserved_amount" - }, - { - "input": "{\"require_coin\":true, \"minimum_balance\":{\"value\": {{reserved_amount}}, \"currency\": {{currency}}}}", - "type": "find_balance", - "output_path": "sender" - }, - { - "input": "{\"operation\":\"subtraction\", \"left_value\": {{sender.balance.value}}, \"right_value\": {{send_buffer}}}", - "type": "math", - "output_path": "available_amount" - }, - { - "input": "{\"minimum\": {{dust_amount}}, \"maximum\": {{available_amount}}}", - "type": "random_number", - "output_path": "recipient_amount" - }, - { - "input": "{\"recipient_amount\":{{recipient_amount}}}", - "type": "print_message" - }, - { - "input": "{\"operation\":\"subtraction\", \"left_value\": {{sender.balance.value}}, \"right_value\": {{recipient_amount}}}", - "type": "math", - "output_path": "total_change_amount" - }, - { - "input": "{\"operation\":\"subtraction\", \"left_value\": {{total_change_amount}}, \"right_value\": {{max_fee_amount}}}", - "type": "math", - "output_path": "change_amount" - }, - { - "input": "{\"change_amount\":{{change_amount}}}", - "type": "print_message" - }, - { - "input": "{\"operation\":\"subtraction\", \"left_value\": \"0\", \"right_value\":{{sender.balance.value}}}", - "type": "math", - "output_path": "sender_amount" - }, - { - "input": "{\"not_account_identifier\":[{{sender.account_identifier}}], \"not_coins\":[{{sender.coin}}], \"minimum_balance\":{\"value\": \"0\", \"currency\": {{currency}}}, \"create_limit\": 100, \"create_probability\": 50}", - "type": "find_balance", - "output_path": "recipient" - }, - { - "input": "\"1\"", - "type": "set_variable", - "output_path": "transfer_dry_run.confirmation_depth" - }, - { - "input": "\"true\"", - "type": "set_variable", - "output_path": "transfer_dry_run.dry_run" - }, - { - "input": "[{\"operation_identifier\":{\"index\":0},\"type\":\"INPUT\",\"account\":{{sender.account_identifier}},\"amount\":{\"value\":{{sender_amount}},\"currency\":{{currency}}}, \"coin_change\":{\"coin_action\":\"coin_spent\", \"coin_identifier\":{{sender.coin}}}},{\"operation_identifier\":{\"index\":1},\"type\":\"OUTPUT\",\"account\":{{recipient.account_identifier}},\"amount\":{\"value\":{{recipient_amount}},\"currency\":{{currency}}}}, {\"operation_identifier\":{\"index\":2},\"type\":\"OUTPUT\",\"account\":{{sender.account_identifier}},\"amount\":{\"value\":{{change_amount}},\"currency\":{{currency}}}}]", - "type": "set_variable", - "output_path": "transfer_dry_run.operations" - }, - { - "input": "{{transfer_dry_run.operations}}", - "type": "print_message" - } - ] - }, - { - "name": "transfer", - "actions": [ - { - "input": "{\"currency\":{{currency}}, \"amounts\":{{transfer_dry_run.suggested_fee}}}", - "type": "find_currency_amount", - "output_path": "suggested_fee" - }, - { - "input": "{\"operation\":\"subtraction\", \"left_value\": {{total_change_amount}}, \"right_value\": {{suggested_fee.value}}}", - "type": "math", - "output_path": "change_amount" - }, - { - "input": "{\"operation\":\"subtraction\", \"left_value\": {{change_amount}}, \"right_value\": {{dust_amount}}}", - "type": "math", - "output_path": "change_minus_dust" - }, - { - "input": "{{change_minus_dust}}", - "type": "assert" - }, - { - "input": "{\"network\":\"Testnet3\", \"blockchain\":\"Bitcoin\"}", - "type": "set_variable", - "output_path": "transfer.network" - }, - { - "input": "\"1\"", - "type": "set_variable", - "output_path": "transfer.confirmation_depth" - }, - { - "input": "[{\"operation_identifier\":{\"index\":0},\"type\":\"INPUT\",\"account\":{{sender.account_identifier}},\"amount\":{\"value\":{{sender_amount}},\"currency\":{{currency}}}, \"coin_change\":{\"coin_action\":\"coin_spent\", \"coin_identifier\":{{sender.coin}}}},{\"operation_identifier\":{\"index\":1},\"type\":\"OUTPUT\",\"account\":{{recipient.account_identifier}},\"amount\":{\"value\":{{recipient_amount}},\"currency\":{{currency}}}}, {\"operation_identifier\":{\"index\":2},\"type\":\"OUTPUT\",\"account\":{{sender.account_identifier}},\"amount\":{\"value\":{{change_amount}},\"currency\":{{currency}}}}]", - "type": "set_variable", - "output_path": "transfer.operations" - }, - { - "input": "{{transfer.operations}}", - "type": "print_message" - } - ] - } - ] - }, - { - "name": "return_funds", - "concurrency": 10, - "scenarios": [ - { - "name": "transfer_dry_run", - "actions": [ - { - "input": "{\"network\":\"Testnet3\", \"blockchain\":\"Bitcoin\"}", - "type": "set_variable", - "output_path": "transfer_dry_run.network" - }, - { - "input": "{\"symbol\":\"tBTC\", \"decimals\":8}", - "type": "set_variable", - "output_path": "currency" - }, - { - "input": "\"1200\"", - "type": "set_variable", - "output_path": "max_fee_amount" - }, - { - "input": "\"1800\"", - "type": "set_variable", - "output_path": "reserved_amount" - }, - { - "input": "{\"require_coin\":true, \"minimum_balance\":{\"value\": {{reserved_amount}}, \"currency\": {{currency}}}}", - "type": "find_balance", - "output_path": "sender" - }, - { - "input": "{\"operation\":\"subtraction\", \"left_value\": {{sender.balance.value}}, \"right_value\": {{max_fee_amount}}}", - "type": "math", - "output_path": "recipient_amount" - }, - { - "input": "{\"recipient_amount\":{{recipient_amount}}}", - "type": "print_message" - }, - { - "input": "{\"operation\":\"subtraction\", \"left_value\": \"0\", \"right_value\":{{sender.balance.value}}}", - "type": "math", - "output_path": "sender_amount" - }, - { - "input": "\"1\"", - "type": "set_variable", - "output_path": "transfer_dry_run.confirmation_depth" - }, - { - "input": "\"true\"", - "type": "set_variable", - "output_path": "transfer_dry_run.dry_run" - }, - { - "input": "{\"address\": \"mkHS9ne12qx9pS9VojpwU5xtRd4T7X7ZUt\"}", - "type": "set_variable", - "output_path": "recipient" - }, - { - "input": "[{\"operation_identifier\":{\"index\":0},\"type\":\"INPUT\",\"account\":{{sender.account_identifier}},\"amount\":{\"value\":{{sender_amount}},\"currency\":{{currency}}}, \"coin_change\":{\"coin_action\":\"coin_spent\", \"coin_identifier\":{{sender.coin}}}},{\"operation_identifier\":{\"index\":1},\"type\":\"OUTPUT\",\"account\":{{recipient}},\"amount\":{\"value\":{{recipient_amount}},\"currency\":{{currency}}}}]", - "type": "set_variable", - "output_path": "transfer_dry_run.operations" - }, - { - "input": "{{transfer_dry_run.operations}}", - "type": "print_message" - } - ] - }, - { - "name": "transfer", - "actions": [ - { - "input": "{\"currency\":{{currency}}, \"amounts\":{{transfer_dry_run.suggested_fee}}}", - "type": "find_currency_amount", - "output_path": "suggested_fee" - }, - { - "input": "{\"operation\":\"subtraction\", \"left_value\": {{sender.balance.value}}, \"right_value\": {{suggested_fee.value}}}", - "type": "math", - "output_path": "recipient_amount" - }, - { - "input": "\"600\"", - "type": "set_variable", - "output_path": "dust_amount" - }, - { - "input": "{\"operation\":\"subtraction\", \"left_value\": {{recipient_amount}}, \"right_value\": {{dust_amount}}}", - "type": "math", - "output_path": "recipient_minus_dust" - }, - { - "input": "{{recipient_minus_dust}}", - "type": "assert" - }, - { - "input": "{\"network\":\"Testnet3\", \"blockchain\":\"Bitcoin\"}", - "type": "set_variable", - "output_path": "transfer.network" - }, - { - "input": "\"1\"", - "type": "set_variable", - "output_path": "transfer.confirmation_depth" - }, - { - "input": "[{\"operation_identifier\":{\"index\":0},\"type\":\"INPUT\",\"account\":{{sender.account_identifier}},\"amount\":{\"value\":{{sender_amount}},\"currency\":{{currency}}}, \"coin_change\":{\"coin_action\":\"coin_spent\", \"coin_identifier\":{{sender.coin}}}},{\"operation_identifier\":{\"index\":1},\"type\":\"OUTPUT\",\"account\":{{recipient}},\"amount\":{\"value\":{{recipient_amount}},\"currency\":{{currency}}}}]", - "type": "set_variable", - "output_path": "transfer.operations" - }, - { - "input": "{{transfer.operations}}", - "type": "print_message" - } - ] - } - ] - } - ], - "end_conditions": { - "create_account": 10, - "transfer": 10 - } - }, - "data": { - "active_reconciliation_concurrency": 0, - "inactive_reconciliation_concurrency": 0, - "inactive_reconciliation_frequency": 0, - "log_blocks": false, - "log_transactions": false, - "log_balance_changes": false, - "log_reconciliations": false, - "ignore_reconciliation_error": false, - "exempt_accounts": "", - "bootstrap_balances": "", - "interesting_accounts": "", - "reconciliation_disabled": false, - "inactive_discrepency_search_disabled": false, - "balance_tracking_disabled": false, - "coin_tracking_disabled": false, - "end_conditions": { - "reconciliation_coverage": { - "coverage":0.95, - "from_tip": true - } - }, - "results_output_file": "" - } -} diff --git a/examples/configuration/ethereum.json b/examples/configuration/ethereum.json deleted file mode 100644 index dcb3c13e..00000000 --- a/examples/configuration/ethereum.json +++ /dev/null @@ -1,55 +0,0 @@ -{ - "network": { - "blockchain": "Ethereum", - "network": "Ropsten" - }, - "online_url": "", - "data_directory": "ethereum-data", - "http_timeout": 300, - "max_retries": 5, - "retry_elapsed_time": 0, - "max_online_connections": 0, - "max_sync_concurrency": 0, - "tip_delay": 60, - "log_configuration": false, - "construction": { - "offline_url": "", - "max_offline_connections": 0, - "stale_depth": 0, - "broadcast_limit": 0, - "ignore_broadcast_failures": false, - "clear_broadcasts": false, - "broadcast_behind_tip": false, - "block_broadcast_limit": 0, - "rebroadcast_all": false, - "constructor_dsl_file": "ethereum.ros", - "end_conditions": { - "create_account": 10, - "transfer": 10 - } - }, - "data": { - "active_reconciliation_concurrency": 0, - "inactive_reconciliation_concurrency": 0, - "inactive_reconciliation_frequency": 0, - "log_blocks": false, - "log_transactions": false, - "log_balance_changes": false, - "log_reconciliations": false, - "ignore_reconciliation_error": false, - "exempt_accounts": "", - "bootstrap_balances": "", - "interesting_accounts": "", - "reconciliation_disabled": false, - "inactive_discrepency_search_disabled": false, - "balance_tracking_disabled": false, - "coin_tracking_disabled": false, - "end_conditions": { - "reconciliation_coverage": { - "coverage":0.95, - "from_tip": true - } - }, - "results_output_file": "" - } -} diff --git a/examples/configuration/ethereum.ros b/examples/configuration/ethereum.ros deleted file mode 100644 index 2f076f07..00000000 --- a/examples/configuration/ethereum.ros +++ /dev/null @@ -1,138 +0,0 @@ -request_funds(1){ - find_account{ - currency = {"symbol":"ETH", "decimals":18}; - random_account = find_balance({ - "minimum_balance":{ - "value": "0", - "currency": {{currency}} - }, - "create_limit":1 - }); - }, - - // Create a separate scenario to request funds so that - // the address we are using to request funds does not - // get rolled back if funds do not yet exist. - request{ - loaded_account = find_balance({ - "account_identifier": {{random_account.account_identifier}}, - "minimum_balance":{ - "value": "10000000000000000", - "currency": {{currency}} - } - }); - } -} - -create_account(1){ - create{ - network = {"network":"Ropsten", "blockchain":"Ethereum"}; - key = generate_key({"curve_type": "secp256k1"}); - account = derive({ - "network_identifier": {{network}}, - "public_key": {{key.public_key}} - }); - - // If the account is not saved, the key will be lost! - save_account({ - "account_identifier": {{account.account_identifier}}, - "keypair": {{key}} - }); - } -} - -transfer(10){ - transfer{ - transfer.network = {"network":"Ropsten", "blockchain":"Ethereum"}; - currency = {"symbol":"ETH", "decimals":18}; - sender = find_balance({ - "minimum_balance":{ - "value": "10000000000000000", - "currency": {{currency}} - } - }); - - // Set the recipient_amount as some value <= sender.balance-max_fee - max_fee = "84000000000000"; - available_amount = {{sender.balance.value}} - {{max_fee}}; - recipient_amount = random_number({"minimum": "1", "maximum": {{available_amount}}}); - print_message({"recipient_amount":{{recipient_amount}}}); - - // Find recipient and construct operations - sender_amount = 0 - {{recipient_amount}}; - recipient = find_balance({ - "not_account_identifier":[{{sender.account_identifier}}], - "minimum_balance":{ - "value": "0", - "currency": {{currency}} - }, - "create_limit": 100, - "create_probability": 50 - }); - transfer.confirmation_depth = "1"; - transfer.operations = [ - { - "operation_identifier":{"index":0}, - "type":"transfer", - "account":{{sender.account_identifier}}, - "amount":{ - "value":{{sender_amount}}, - "currency":{{currency}} - } - }, - { - "operation_identifier":{"index":1}, - "type":"transfer", - "account":{{recipient.account_identifier}}, - "amount":{ - "value":{{recipient_amount}}, - "currency":{{currency}} - } - } - ]; - } -} - -return_funds(10){ - // TODO: add suggested fee dry run to ensure account fully cleared - transfer{ - transfer.network = {"network":"Ropsten", "blockchain":"Ethereum"}; - currency = {"symbol":"ETH", "decimals":18}; - max_fee = "84000000000000"; - sender = find_balance({ - "minimum_balance":{ - "value": {{max_fee}}, - "currency": {{currency}} - } - }); - - // Set the recipient_amount as some sender.balance-max_fee - available_amount = {{sender.balance.value}} - {{max_fee}}; - print_message({"available_amount":{{available_amount}}}); - sender_amount = 0 - {{available_amount}}; - - // Provide a static address as the recipient and construct operations - faucet = {"address":"0xb41B39479a525AB69e38c701A713D98E3074252c"}; - transfer.confirmation_depth = "1"; - transfer.operations = [ - { - "operation_identifier":{"index":0}, - "type":"transfer", - "account":{{sender.account_identifier}}, - "amount":{ - "value":{{sender_amount}}, - "currency":{{currency}} - } - }, - { - "operation_identifier":{"index":1}, - "type":"transfer", - "account":{{faucet}}, - "amount":{ - "value":{{available_amount}}, - "currency":{{currency}} - } - } - ]; - } -} From fbf339d97c392e91675e411a764016b32b773152 Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Tue, 27 Oct 2020 15:55:37 -0700 Subject: [PATCH 160/376] Update example links --- README.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 30f90e09..44055700 100644 --- a/README.md +++ b/README.md @@ -89,8 +89,8 @@ You can view a full list of all configuration options [here](https://pkg.go.dev/ In the `examples/configuration` directory, you can find examples configuration files for running tests against a Bitcoin Rosetta implementation -([config](examples/configuration/bitcoin.json)) and an Ethereum Rosetta -implementation ([config](examples/configuration/ethereum.json)). +([config](https://github.com/coinbase/rosetta-bitcoin/tree/master/rosetta-cli-conf)) and an Ethereum Rosetta +implementation ([config](https://github.com/coinbase/rosetta-ethereum/tree/master/rosetta-cli-conf)). #### Writing check:construction Tests The new Construction API testing framework (first released in `rosetta-cli@v0.5.0`) uses @@ -99,8 +99,8 @@ You can read more about the design goals [here](https://community.rosetta-api.or Most teams write their Construction API tests using the [Rosetta Constructor DSL](https://github.com/coinbase/rosetta-sdk-go/tree/master/constructor/dsl). -You can find an example of a file written in this DSL in the examples -folder ([here](https://github.com/coinbase/rosetta-cli/blob/master/examples/configuration/ethereum.ros)). +We have examples of a DSL files written for [UTXO-based chains](https://github.com/coinbase/rosetta-bitcoin/blob/master/rosetta-cli-conf/testnet/bitcoin.ros) +and [account-based chains](https://github.com/coinbase/rosetta-ethereum/blob/master/rosetta-cli-conf/testnet/ethereum.ros). ##### Terminology When first learning about a new topic, it is often useful to understand the From 675fde16ea11c239ba64e828d8f6e6419a2d31d9 Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Wed, 28 Oct 2020 12:16:14 -0700 Subject: [PATCH 161/376] Update rosetta-sdk-go --- go.mod | 2 +- go.sum | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index 2ded3f9c..ef38d591 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/coinbase/rosetta-cli go 1.13 require ( - github.com/coinbase/rosetta-sdk-go v0.5.8-0.20201027222031-dd9e29377d5f + github.com/coinbase/rosetta-sdk-go v0.5.8 github.com/fatih/color v1.9.0 github.com/olekukonko/tablewriter v0.0.2-0.20190409134802-7e037d187b0c github.com/spf13/cobra v1.1.1 diff --git a/go.sum b/go.sum index 47fae483..73c37f7f 100644 --- a/go.sum +++ b/go.sum @@ -80,6 +80,8 @@ github.com/coinbase/rosetta-sdk-go v0.5.7 h1:BaR/+O3GzrsyunVNkVQHtjDCcId8G1Fh/Rq github.com/coinbase/rosetta-sdk-go v0.5.7/go.mod h1:l5aNeyeZKBkmWbVdkdLpWdToQ6hTwI7cZ1OU9cMbljY= github.com/coinbase/rosetta-sdk-go v0.5.8-0.20201027222031-dd9e29377d5f h1:aWkN9dKMkMMpZKX5QycpePxH176Fj2fNNC7jESfLZw0= github.com/coinbase/rosetta-sdk-go v0.5.8-0.20201027222031-dd9e29377d5f/go.mod h1:l5aNeyeZKBkmWbVdkdLpWdToQ6hTwI7cZ1OU9cMbljY= +github.com/coinbase/rosetta-sdk-go v0.5.8 h1:Sf7iQPjexIsa7zQfx0PdRIlfsjrDPfBuzyxbJogZDqw= +github.com/coinbase/rosetta-sdk-go v0.5.8/go.mod h1:xd4wYUhV3LkY78SPH8BUhc88rXfn2jYgN9BfiSjbcvM= github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= @@ -375,6 +377,8 @@ github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljT github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= github.com/vmihailenco/msgpack/v5 v5.0.0-beta.8 h1:R2L6zPq1pWFumpeIxAJoeiov5GxyEZUq9NyS8eus/6s= github.com/vmihailenco/msgpack/v5 v5.0.0-beta.8/go.mod h1:HVxBVPUK/+fZMonk4bi1islLa8V3cfnBug0+4dykPzo= +github.com/vmihailenco/msgpack/v5 v5.0.0-beta.9 h1:iBRIniTnWOo0kqkg3k3XR8Vn6OCkVlIuZNo0UoBrKx4= +github.com/vmihailenco/msgpack/v5 v5.0.0-beta.9/go.mod h1:HVxBVPUK/+fZMonk4bi1islLa8V3cfnBug0+4dykPzo= github.com/vmihailenco/tagparser v0.1.2 h1:gnjoVuB/kljJ5wICEEOpx98oXMWPLj22G67Vbd1qPqc= github.com/vmihailenco/tagparser v0.1.2/go.mod h1:OeAg3pn3UbLjkWt+rN9oFYB6u/cQgqMEUPoW2WPyhdI= github.com/wsddn/go-ecdh v0.0.0-20161211032359-48726bab9208/go.mod h1:IotVbo4F+mw0EzQ08zFqg7pK3FebNXpaMsRy2RT+Ees= From d915e936cc844cf9c095f0909d7c66169657a229 Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Wed, 28 Oct 2020 12:19:01 -0700 Subject: [PATCH 162/376] Update version --- cmd/root.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/root.go b/cmd/root.go index a7ee8995..b5eaf798 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -257,6 +257,6 @@ var versionCmd = &cobra.Command{ Use: "version", Short: "Print rosetta-cli version", Run: func(cmd *cobra.Command, args []string) { - fmt.Println("v0.5.17") + fmt.Println("v0.5.18") }, } From f9e77920c1133fb788c0557d0f384602e8ca5d24 Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Thu, 29 Oct 2020 13:44:13 -0700 Subject: [PATCH 163/376] Get rosetta-sdk-go fix for reconciler --- go.mod | 2 +- go.sum | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index ef38d591..a2fa5a9a 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/coinbase/rosetta-cli go 1.13 require ( - github.com/coinbase/rosetta-sdk-go v0.5.8 + github.com/coinbase/rosetta-sdk-go v0.5.9-0.20201029204210-ce925f00ccc8 github.com/fatih/color v1.9.0 github.com/olekukonko/tablewriter v0.0.2-0.20190409134802-7e037d187b0c github.com/spf13/cobra v1.1.1 diff --git a/go.sum b/go.sum index 73c37f7f..51a48f21 100644 --- a/go.sum +++ b/go.sum @@ -82,6 +82,8 @@ github.com/coinbase/rosetta-sdk-go v0.5.8-0.20201027222031-dd9e29377d5f h1:aWkN9 github.com/coinbase/rosetta-sdk-go v0.5.8-0.20201027222031-dd9e29377d5f/go.mod h1:l5aNeyeZKBkmWbVdkdLpWdToQ6hTwI7cZ1OU9cMbljY= github.com/coinbase/rosetta-sdk-go v0.5.8 h1:Sf7iQPjexIsa7zQfx0PdRIlfsjrDPfBuzyxbJogZDqw= github.com/coinbase/rosetta-sdk-go v0.5.8/go.mod h1:xd4wYUhV3LkY78SPH8BUhc88rXfn2jYgN9BfiSjbcvM= +github.com/coinbase/rosetta-sdk-go v0.5.9-0.20201029204210-ce925f00ccc8 h1:j+G5/Hy411/jsh12rVYMjw7VLEITqLVrzYhB9AOYElg= +github.com/coinbase/rosetta-sdk-go v0.5.9-0.20201029204210-ce925f00ccc8/go.mod h1:xd4wYUhV3LkY78SPH8BUhc88rXfn2jYgN9BfiSjbcvM= github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= From 6c228d68f4d42e278286dd775421d26add61b804 Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Thu, 29 Oct 2020 13:53:37 -0700 Subject: [PATCH 164/376] canonical block transactional --- go.mod | 2 +- go.sum | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index a2fa5a9a..96bb7ddd 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/coinbase/rosetta-cli go 1.13 require ( - github.com/coinbase/rosetta-sdk-go v0.5.9-0.20201029204210-ce925f00ccc8 + github.com/coinbase/rosetta-sdk-go v0.5.9-0.20201029205252-2500d4047a2f github.com/fatih/color v1.9.0 github.com/olekukonko/tablewriter v0.0.2-0.20190409134802-7e037d187b0c github.com/spf13/cobra v1.1.1 diff --git a/go.sum b/go.sum index 51a48f21..6d768b0f 100644 --- a/go.sum +++ b/go.sum @@ -84,6 +84,8 @@ github.com/coinbase/rosetta-sdk-go v0.5.8 h1:Sf7iQPjexIsa7zQfx0PdRIlfsjrDPfBuzyx github.com/coinbase/rosetta-sdk-go v0.5.8/go.mod h1:xd4wYUhV3LkY78SPH8BUhc88rXfn2jYgN9BfiSjbcvM= github.com/coinbase/rosetta-sdk-go v0.5.9-0.20201029204210-ce925f00ccc8 h1:j+G5/Hy411/jsh12rVYMjw7VLEITqLVrzYhB9AOYElg= github.com/coinbase/rosetta-sdk-go v0.5.9-0.20201029204210-ce925f00ccc8/go.mod h1:xd4wYUhV3LkY78SPH8BUhc88rXfn2jYgN9BfiSjbcvM= +github.com/coinbase/rosetta-sdk-go v0.5.9-0.20201029205252-2500d4047a2f h1:JaNGbt+C3jHYYWJ7+wcSuOH+n7CVXwNYPEYe0O87mSg= +github.com/coinbase/rosetta-sdk-go v0.5.9-0.20201029205252-2500d4047a2f/go.mod h1:xd4wYUhV3LkY78SPH8BUhc88rXfn2jYgN9BfiSjbcvM= github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= From 61c839ac4fd755cbf721df694a07868b5899f743 Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Thu, 29 Oct 2020 13:56:22 -0700 Subject: [PATCH 165/376] Fix processor errors --- pkg/processor/balance_storage_handler.go | 6 ++--- pkg/processor/balance_storage_helper.go | 16 +++++++++---- pkg/processor/balance_storage_helper_test.go | 9 ++++--- pkg/processor/reconciler_handler.go | 4 ++-- pkg/processor/reconciler_helper.go | 25 +++++++++++++++----- 5 files changed, 39 insertions(+), 21 deletions(-) diff --git a/pkg/processor/balance_storage_handler.go b/pkg/processor/balance_storage_handler.go index b7623b4d..0e7038fb 100644 --- a/pkg/processor/balance_storage_handler.go +++ b/pkg/processor/balance_storage_handler.go @@ -35,7 +35,7 @@ type BalanceStorageHandler struct { reconciler *reconciler.Reconciler reconcile bool - interestingAccount *reconciler.AccountCurrency + interestingAccount *types.AccountCurrency } // NewBalanceStorageHandler returns a new *BalanceStorageHandler. @@ -43,7 +43,7 @@ func NewBalanceStorageHandler( logger *logger.Logger, reconciler *reconciler.Reconciler, reconcile bool, - interestingAccount *reconciler.AccountCurrency, + interestingAccount *types.AccountCurrency, ) *BalanceStorageHandler { return &BalanceStorageHandler{ logger: logger, @@ -73,7 +73,7 @@ func (h *BalanceStorageHandler) BlockAdded( if h.interestingAccount != nil { var interestingChange *parser.BalanceChange for _, change := range changes { - if types.Hash(&reconciler.AccountCurrency{ + if types.Hash(&types.AccountCurrency{ Account: change.Account, Currency: change.Currency, }) == types.Hash(h.interestingAccount) { diff --git a/pkg/processor/balance_storage_helper.go b/pkg/processor/balance_storage_helper.go index 99b737ff..a925b14d 100644 --- a/pkg/processor/balance_storage_helper.go +++ b/pkg/processor/balance_storage_helper.go @@ -21,8 +21,8 @@ import ( "github.com/coinbase/rosetta-sdk-go/asserter" "github.com/coinbase/rosetta-sdk-go/fetcher" "github.com/coinbase/rosetta-sdk-go/parser" - "github.com/coinbase/rosetta-sdk-go/reconciler" "github.com/coinbase/rosetta-sdk-go/storage" + "github.com/coinbase/rosetta-sdk-go/syncer" "github.com/coinbase/rosetta-sdk-go/types" "github.com/coinbase/rosetta-sdk-go/utils" ) @@ -51,7 +51,7 @@ func NewBalanceStorageHelper( network *types.NetworkIdentifier, fetcher *fetcher.Fetcher, lookupBalanceByBlock bool, - exemptAccounts []*reconciler.AccountCurrency, + exemptAccounts []*types.AccountCurrency, interestingOnly bool, balanceExemptions []*types.BalanceExemption, initialFetchDisabled bool, @@ -96,18 +96,24 @@ func (h *BalanceStorageHelper) AccountBalance( // In the case that we are syncing from arbitrary height, // we may need to recover the balance of an account to // perform validations. - amount, _, _, err := utils.CurrencyBalance( + amount, retrievedBlock, _, err := utils.CurrencyBalance( ctx, h.network, h.fetcher, account, currency, - block, + block.Index, ) if err != nil { return nil, fmt.Errorf("%w: unable to get currency balance", err) } + // If the returned balance block does not match the intended + // block a re-org could've occured. + if types.Hash(block) != types.Hash(retrievedBlock) { + return nil, syncer.ErrOrphanHead + } + return &types.Amount{ Value: amount.Value, Currency: currency, @@ -134,7 +140,7 @@ func (h *BalanceStorageHelper) ExemptFunc() parser.ExemptOperation { } } - thisAcct := types.Hash(&reconciler.AccountCurrency{ + thisAcct := types.Hash(&types.AccountCurrency{ Account: op.Account, Currency: op.Amount.Currency, }) diff --git a/pkg/processor/balance_storage_helper_test.go b/pkg/processor/balance_storage_helper_test.go index 55aaabae..928efe4f 100644 --- a/pkg/processor/balance_storage_helper_test.go +++ b/pkg/processor/balance_storage_helper_test.go @@ -17,13 +17,12 @@ package processor import ( "testing" - "github.com/coinbase/rosetta-sdk-go/reconciler" "github.com/coinbase/rosetta-sdk-go/types" "github.com/stretchr/testify/assert" ) var ( - opAmountCurrency = &reconciler.AccountCurrency{ + opAmountCurrency = &types.AccountCurrency{ Account: &types.AccountIdentifier{ Address: "hello", }, @@ -36,12 +35,12 @@ var ( func TestExemptFuncExemptAccounts(t *testing.T) { var tests = map[string]struct { - exemptAccounts []*reconciler.AccountCurrency + exemptAccounts []*types.AccountCurrency exempt bool }{ "no exempt accounts": {}, "account not exempt": { - exemptAccounts: []*reconciler.AccountCurrency{ + exemptAccounts: []*types.AccountCurrency{ { Account: &types.AccountIdentifier{ Address: "addr1", @@ -57,7 +56,7 @@ func TestExemptFuncExemptAccounts(t *testing.T) { }, }, "account is exempt": { - exemptAccounts: []*reconciler.AccountCurrency{ + exemptAccounts: []*types.AccountCurrency{ { Account: &types.AccountIdentifier{ Address: "addr1", diff --git a/pkg/processor/reconciler_handler.go b/pkg/processor/reconciler_handler.go index fa228a65..2d74ba11 100644 --- a/pkg/processor/reconciler_handler.go +++ b/pkg/processor/reconciler_handler.go @@ -36,7 +36,7 @@ type ReconcilerHandler struct { balanceStorage *storage.BalanceStorage haltOnReconciliationError bool - InactiveFailure *reconciler.AccountCurrency + InactiveFailure *types.AccountCurrency InactiveFailureBlock *types.BlockIdentifier ActiveFailureBlock *types.BlockIdentifier @@ -88,7 +88,7 @@ func (h *ReconcilerHandler) ReconciliationFailed( if reconciliationType == reconciler.InactiveReconciliation { // Populate inactive failure information so we can try to find block with // missing ops. - h.InactiveFailure = &reconciler.AccountCurrency{ + h.InactiveFailure = &types.AccountCurrency{ Account: account, Currency: currency, } diff --git a/pkg/processor/reconciler_helper.go b/pkg/processor/reconciler_helper.go index bd1bcf77..ddcee94f 100644 --- a/pkg/processor/reconciler_helper.go +++ b/pkg/processor/reconciler_helper.go @@ -32,6 +32,7 @@ type ReconcilerHelper struct { network *types.NetworkIdentifier fetcher *fetcher.Fetcher + database storage.Database blockStorage *storage.BlockStorage balanceStorage *storage.BalanceStorage } @@ -40,26 +41,36 @@ type ReconcilerHelper struct { func NewReconcilerHelper( network *types.NetworkIdentifier, fetcher *fetcher.Fetcher, + database storage.Database, blockStorage *storage.BlockStorage, balanceStorage *storage.BalanceStorage, ) *ReconcilerHelper { return &ReconcilerHelper{ network: network, fetcher: fetcher, + database: database, blockStorage: blockStorage, balanceStorage: balanceStorage, } } +// DatabaseTransaction returns a new read-only storage.DatabaseTransaction. +func (h *ReconcilerHelper) DatabaseTransaction( + ctx context.Context, +) storage.DatabaseTransaction { + return h.database.NewDatabaseTransaction(ctx, false) +} + // CanonicalBlock returns a boolean indicating if a block // is in the canonical chain. This is necessary to reconcile across // reorgs. If the block returned on an account balance fetch // does not exist, reconciliation will be skipped. func (h *ReconcilerHelper) CanonicalBlock( ctx context.Context, + dbTx storage.DatabaseTransaction, block *types.BlockIdentifier, ) (bool, error) { - return h.blockStorage.CanonicalBlock(ctx, block) + return h.blockStorage.CanonicalBlockTransactional(ctx, block, dbTx) } // CurrentBlock returns the last processed block and is used @@ -67,8 +78,9 @@ func (h *ReconcilerHelper) CanonicalBlock( // inactive reconciliation. func (h *ReconcilerHelper) CurrentBlock( ctx context.Context, + dbTx storage.DatabaseTransaction, ) (*types.BlockIdentifier, error) { - return h.blockStorage.GetHeadBlockIdentifier(ctx) + return h.blockStorage.GetHeadBlockIdentifierTransactional(ctx, dbTx) } // ComputedBalance returns the balance of an account in block storage. @@ -76,11 +88,12 @@ func (h *ReconcilerHelper) CurrentBlock( // package to allow for separation from a default storage backend. func (h *ReconcilerHelper) ComputedBalance( ctx context.Context, + dbTx storage.DatabaseTransaction, account *types.AccountIdentifier, currency *types.Currency, - headBlock *types.BlockIdentifier, + index int64, ) (*types.Amount, error) { - return h.balanceStorage.GetOrSetBalance(ctx, account, currency, headBlock) + return h.balanceStorage.GetBalanceTransactional(ctx, dbTx, account, currency, index) } // LiveBalance returns the live balance of an account. @@ -88,7 +101,7 @@ func (h *ReconcilerHelper) LiveBalance( ctx context.Context, account *types.AccountIdentifier, currency *types.Currency, - headBlock *types.BlockIdentifier, + index int64, ) (*types.Amount, *types.BlockIdentifier, error) { amt, block, _, err := utils.CurrencyBalance( ctx, @@ -96,7 +109,7 @@ func (h *ReconcilerHelper) LiveBalance( h.fetcher, account, currency, - headBlock, + index, ) if err != nil { return nil, nil, err From 8b0a78f605c00a0af78dc3c26b380f1fa9d44f17 Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Thu, 29 Oct 2020 13:58:35 -0700 Subject: [PATCH 166/376] Update reconciler init --- pkg/processor/balance_storage_helper.go | 2 +- pkg/tester/data.go | 18 ++++++++++-------- 2 files changed, 11 insertions(+), 9 deletions(-) diff --git a/pkg/processor/balance_storage_helper.go b/pkg/processor/balance_storage_helper.go index a925b14d..30006017 100644 --- a/pkg/processor/balance_storage_helper.go +++ b/pkg/processor/balance_storage_helper.go @@ -109,7 +109,7 @@ func (h *BalanceStorageHelper) AccountBalance( } // If the returned balance block does not match the intended - // block a re-org could've occured. + // block a re-org could've occurred. if types.Hash(block) != types.Hash(retrievedBlock) { return nil, syncer.ErrOrphanHead } diff --git a/pkg/tester/data.go b/pkg/tester/data.go index 23ff0892..9be03742 100644 --- a/pkg/tester/data.go +++ b/pkg/tester/data.go @@ -103,14 +103,14 @@ func shouldReconcile(config *configuration.Configuration) bool { return true } -// loadAccounts is a utility function to parse the []*reconciler.AccountCurrency +// loadAccounts is a utility function to parse the []*types.AccountCurrency // in a file. -func loadAccounts(filePath string) ([]*reconciler.AccountCurrency, error) { +func loadAccounts(filePath string) ([]*types.AccountCurrency, error) { if len(filePath) == 0 { - return []*reconciler.AccountCurrency{}, nil + return []*types.AccountCurrency{}, nil } - accounts := []*reconciler.AccountCurrency{} + accounts := []*types.AccountCurrency{} if err := utils.LoadAndParse(filePath, &accounts); err != nil { return nil, fmt.Errorf("%w: unable to open account file", err) } @@ -140,7 +140,7 @@ func InitializeData( fetcher *fetcher.Fetcher, cancel context.CancelFunc, genesisBlock *types.BlockIdentifier, - interestingAccount *reconciler.AccountCurrency, + interestingAccount *types.AccountCurrency, signalReceived *bool, ) *DataTester { dataPath, err := utils.CreateCommandPath(config.DataDirectory, dataCmdName, network) @@ -208,6 +208,7 @@ func InitializeData( reconcilerHelper := processor.NewReconcilerHelper( network, fetcher, + localStore, blockStorage, balanceStorage, ) @@ -867,7 +868,7 @@ func (t *DataTester) HandleErr(err error, sigListeners *[]context.CancelFunc) er // FindMissingOps logs the types.BlockIdentifier of a block // that is missing balance-changing operations for a -// *reconciler.AccountCurrency. +// *types.AccountCurrency. func (t *DataTester) FindMissingOps( ctx context.Context, originalErr error, @@ -913,7 +914,7 @@ func (t *DataTester) FindMissingOps( func (t *DataTester) recursiveOpSearch( ctx context.Context, sigListeners *[]context.CancelFunc, - accountCurrency *reconciler.AccountCurrency, + accountCurrency *types.AccountCurrency, startIndex int64, endIndex int64, ) (*types.BlockIdentifier, error) { @@ -948,6 +949,7 @@ func (t *DataTester) recursiveOpSearch( reconcilerHelper := processor.NewReconcilerHelper( t.network, t.fetcher, + localStore, blockStorage, balanceStorage, ) @@ -973,7 +975,7 @@ func (t *DataTester) recursiveOpSearch( // operations. reconciler.WithInactiveConcurrency(0), reconciler.WithLookupBalanceByBlock(), - reconciler.WithInterestingAccounts([]*reconciler.AccountCurrency{accountCurrency}), + reconciler.WithInterestingAccounts([]*types.AccountCurrency{accountCurrency}), ) balanceStorageHelper := processor.NewBalanceStorageHelper( From cb3f46e9af59fc4a0a4450da451942e9753bfe06 Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Thu, 29 Oct 2020 14:08:56 -0700 Subject: [PATCH 167/376] Fix stats printing when start index provided --- pkg/results/data_results.go | 22 +++++++++++++++++++--- pkg/tester/data.go | 2 ++ 2 files changed, 21 insertions(+), 3 deletions(-) diff --git a/pkg/results/data_results.go b/pkg/results/data_results.go index 807bbaf8..bf7f5c71 100644 --- a/pkg/results/data_results.go +++ b/pkg/results/data_results.go @@ -280,6 +280,7 @@ func ComputeCheckDataProgress( fetcher *fetcher.Fetcher, network *types.NetworkIdentifier, counters *storage.CounterStorage, + blockStorage *storage.BlockStorage, reconciler *reconciler.Reconciler, ) *CheckDataProgress { networkStatus, fetchErr := fetcher.NetworkStatusRetry(ctx, network, nil) @@ -289,6 +290,17 @@ func ComputeCheckDataProgress( } tipIndex := networkStatus.CurrentBlockIdentifier.Index + // Get current tip in the case that re-orgs occurred + // or a custom start index was provied. + headBlock, err := blockStorage.GetHeadBlockIdentifier(ctx) + if errors.Is(err, storage.ErrHeadBlockNotFound) { + return nil + } + if err != nil { + fmt.Printf("%s: cannot get head block", err.Error()) + return nil + } + blocks, err := counters.Get(ctx, storage.BlockCounter) if err != nil { fmt.Printf("%s: cannot get block counter", err.Error()) @@ -305,6 +317,8 @@ func ComputeCheckDataProgress( return nil } + // adjustedBlocks is used to calculate the sync rate (regardless + // of which block we started syncing at) adjustedBlocks := blocks.Int64() - orphans.Int64() if tipIndex-adjustedBlocks <= 0 { // return if no blocks to sync return nil @@ -322,15 +336,15 @@ func ComputeCheckDataProgress( blocksPerSecond := new(big.Float).Quo(new(big.Float).SetInt64(adjustedBlocks), new(big.Float).SetInt(elapsedTime)) blocksPerSecondFloat, _ := blocksPerSecond.Float64() - blocksSynced := new(big.Float).Quo(new(big.Float).SetInt64(adjustedBlocks), new(big.Float).SetInt64(tipIndex)) + blocksSynced := new(big.Float).Quo(new(big.Float).SetInt64(headBlock.Index), new(big.Float).SetInt64(tipIndex)) blocksSyncedFloat, _ := blocksSynced.Float64() return &CheckDataProgress{ - Blocks: adjustedBlocks, + Blocks: headBlock.Index, Tip: tipIndex, Completed: blocksSyncedFloat * utils.OneHundred, Rate: blocksPerSecondFloat, - TimeRemaining: utils.TimeToTip(blocksPerSecondFloat, adjustedBlocks, tipIndex).String(), + TimeRemaining: utils.TimeToTip(blocksPerSecondFloat, headBlock.Index, tipIndex).String(), ReconcilerQueueSize: reconciler.QueueSize(), ReconcilerLastIndex: reconciler.LastIndexReconciled(), } @@ -347,6 +361,7 @@ type CheckDataStatus struct { // *CheckDataStatus. func ComputeCheckDataStatus( ctx context.Context, + blocks *storage.BlockStorage, counters *storage.CounterStorage, balances *storage.BalanceStorage, fetcher *fetcher.Fetcher, @@ -364,6 +379,7 @@ func ComputeCheckDataStatus( fetcher, network, counters, + blocks, reconciler, ), } diff --git a/pkg/tester/data.go b/pkg/tester/data.go index 9be03742..be3983d0 100644 --- a/pkg/tester/data.go +++ b/pkg/tester/data.go @@ -418,6 +418,7 @@ func (t *DataTester) StartPeriodicLogger( status := results.ComputeCheckDataStatus( ctx, + t.blockStorage, t.counterStorage, t.balanceStorage, t.fetcher, @@ -436,6 +437,7 @@ func (t *DataTester) ServeHTTP(w http.ResponseWriter, r *http.Request) { status := results.ComputeCheckDataStatus( r.Context(), + t.blockStorage, t.counterStorage, t.balanceStorage, t.fetcher, From b6e153090f6e6d789cd60291a819ea71ecd9957d Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Thu, 29 Oct 2020 14:10:11 -0700 Subject: [PATCH 168/376] Update to master of rosetta-sdk-go --- go.mod | 2 +- go.sum | 14 ++------------ 2 files changed, 3 insertions(+), 13 deletions(-) diff --git a/go.mod b/go.mod index 96bb7ddd..ada68e65 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/coinbase/rosetta-cli go 1.13 require ( - github.com/coinbase/rosetta-sdk-go v0.5.9-0.20201029205252-2500d4047a2f + github.com/coinbase/rosetta-sdk-go v0.5.9-0.20201029210921-d7499a34c1f6 github.com/fatih/color v1.9.0 github.com/olekukonko/tablewriter v0.0.2-0.20190409134802-7e037d187b0c github.com/spf13/cobra v1.1.1 diff --git a/go.sum b/go.sum index 6d768b0f..5e99c3ee 100644 --- a/go.sum +++ b/go.sum @@ -76,16 +76,8 @@ github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+ github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cloudflare/cloudflare-go v0.10.2-0.20190916151808-a80f83b9add9/go.mod h1:1MxXX1Ux4x6mqPmjkUgTP1CdXIBXKX7T+Jk9Gxrmx+U= -github.com/coinbase/rosetta-sdk-go v0.5.7 h1:BaR/+O3GzrsyunVNkVQHtjDCcId8G1Fh/RqEbeyExnk= -github.com/coinbase/rosetta-sdk-go v0.5.7/go.mod h1:l5aNeyeZKBkmWbVdkdLpWdToQ6hTwI7cZ1OU9cMbljY= -github.com/coinbase/rosetta-sdk-go v0.5.8-0.20201027222031-dd9e29377d5f h1:aWkN9dKMkMMpZKX5QycpePxH176Fj2fNNC7jESfLZw0= -github.com/coinbase/rosetta-sdk-go v0.5.8-0.20201027222031-dd9e29377d5f/go.mod h1:l5aNeyeZKBkmWbVdkdLpWdToQ6hTwI7cZ1OU9cMbljY= -github.com/coinbase/rosetta-sdk-go v0.5.8 h1:Sf7iQPjexIsa7zQfx0PdRIlfsjrDPfBuzyxbJogZDqw= -github.com/coinbase/rosetta-sdk-go v0.5.8/go.mod h1:xd4wYUhV3LkY78SPH8BUhc88rXfn2jYgN9BfiSjbcvM= -github.com/coinbase/rosetta-sdk-go v0.5.9-0.20201029204210-ce925f00ccc8 h1:j+G5/Hy411/jsh12rVYMjw7VLEITqLVrzYhB9AOYElg= -github.com/coinbase/rosetta-sdk-go v0.5.9-0.20201029204210-ce925f00ccc8/go.mod h1:xd4wYUhV3LkY78SPH8BUhc88rXfn2jYgN9BfiSjbcvM= -github.com/coinbase/rosetta-sdk-go v0.5.9-0.20201029205252-2500d4047a2f h1:JaNGbt+C3jHYYWJ7+wcSuOH+n7CVXwNYPEYe0O87mSg= -github.com/coinbase/rosetta-sdk-go v0.5.9-0.20201029205252-2500d4047a2f/go.mod h1:xd4wYUhV3LkY78SPH8BUhc88rXfn2jYgN9BfiSjbcvM= +github.com/coinbase/rosetta-sdk-go v0.5.9-0.20201029210921-d7499a34c1f6 h1:5zDhG7QoXRf2GtNeDqfnUdAe5gm1fdKy0h5Txa7NHec= +github.com/coinbase/rosetta-sdk-go v0.5.9-0.20201029210921-d7499a34c1f6/go.mod h1:xd4wYUhV3LkY78SPH8BUhc88rXfn2jYgN9BfiSjbcvM= github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= @@ -379,8 +371,6 @@ github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1 github.com/tyler-smith/go-bip39 v1.0.1-0.20181017060643-dbb3b84ba2ef/go.mod h1:sJ5fKU0s6JVwZjjcUEX2zFOnvq0ASQ2K9Zr6cf67kNs= github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= -github.com/vmihailenco/msgpack/v5 v5.0.0-beta.8 h1:R2L6zPq1pWFumpeIxAJoeiov5GxyEZUq9NyS8eus/6s= -github.com/vmihailenco/msgpack/v5 v5.0.0-beta.8/go.mod h1:HVxBVPUK/+fZMonk4bi1islLa8V3cfnBug0+4dykPzo= github.com/vmihailenco/msgpack/v5 v5.0.0-beta.9 h1:iBRIniTnWOo0kqkg3k3XR8Vn6OCkVlIuZNo0UoBrKx4= github.com/vmihailenco/msgpack/v5 v5.0.0-beta.9/go.mod h1:HVxBVPUK/+fZMonk4bi1islLa8V3cfnBug0+4dykPzo= github.com/vmihailenco/tagparser v0.1.2 h1:gnjoVuB/kljJ5wICEEOpx98oXMWPLj22G67Vbd1qPqc= From 89927cad52193e11cf2c2b8f138359af80b924f1 Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Thu, 29 Oct 2020 14:10:40 -0700 Subject: [PATCH 169/376] Update version --- cmd/root.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/root.go b/cmd/root.go index b5eaf798..d89c144f 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -257,6 +257,6 @@ var versionCmd = &cobra.Command{ Use: "version", Short: "Print rosetta-cli version", Run: func(cmd *cobra.Command, args []string) { - fmt.Println("v0.5.18") + fmt.Println("v0.5.19") }, } From 46623009eb07f9ac606e599323130608777dbdc1 Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Thu, 29 Oct 2020 14:15:48 -0700 Subject: [PATCH 170/376] nits --- pkg/processor/balance_storage_helper.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/pkg/processor/balance_storage_helper.go b/pkg/processor/balance_storage_helper.go index 30006017..7374b0f6 100644 --- a/pkg/processor/balance_storage_helper.go +++ b/pkg/processor/balance_storage_helper.go @@ -84,7 +84,7 @@ func (h *BalanceStorageHelper) AccountBalance( ctx context.Context, account *types.AccountIdentifier, currency *types.Currency, - block *types.BlockIdentifier, + lookupBlock *types.BlockIdentifier, ) (*types.Amount, error) { if !h.lookupBalanceByBlock || h.initialFetchDisabled { return &types.Amount{ @@ -96,13 +96,13 @@ func (h *BalanceStorageHelper) AccountBalance( // In the case that we are syncing from arbitrary height, // we may need to recover the balance of an account to // perform validations. - amount, retrievedBlock, _, err := utils.CurrencyBalance( + amount, block, _, err := utils.CurrencyBalance( ctx, h.network, h.fetcher, account, currency, - block.Index, + lookupBlock.Index, ) if err != nil { return nil, fmt.Errorf("%w: unable to get currency balance", err) @@ -110,7 +110,7 @@ func (h *BalanceStorageHelper) AccountBalance( // If the returned balance block does not match the intended // block a re-org could've occurred. - if types.Hash(block) != types.Hash(retrievedBlock) { + if types.Hash(lookupBlock) != types.Hash(block) { return nil, syncer.ErrOrphanHead } From 656ae5a68a4336693b3540686b86a8c58fdc9c04 Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Fri, 30 Oct 2020 14:36:12 -0700 Subject: [PATCH 171/376] Update rosetta-sdk-go --- go.mod | 2 +- go.sum | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index ada68e65..bda12ee1 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/coinbase/rosetta-cli go 1.13 require ( - github.com/coinbase/rosetta-sdk-go v0.5.9-0.20201029210921-d7499a34c1f6 + github.com/coinbase/rosetta-sdk-go v0.5.9-0.20201030213420-e8c5df9fe202 github.com/fatih/color v1.9.0 github.com/olekukonko/tablewriter v0.0.2-0.20190409134802-7e037d187b0c github.com/spf13/cobra v1.1.1 diff --git a/go.sum b/go.sum index 5e99c3ee..219dc29f 100644 --- a/go.sum +++ b/go.sum @@ -78,6 +78,8 @@ github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDk github.com/cloudflare/cloudflare-go v0.10.2-0.20190916151808-a80f83b9add9/go.mod h1:1MxXX1Ux4x6mqPmjkUgTP1CdXIBXKX7T+Jk9Gxrmx+U= github.com/coinbase/rosetta-sdk-go v0.5.9-0.20201029210921-d7499a34c1f6 h1:5zDhG7QoXRf2GtNeDqfnUdAe5gm1fdKy0h5Txa7NHec= github.com/coinbase/rosetta-sdk-go v0.5.9-0.20201029210921-d7499a34c1f6/go.mod h1:xd4wYUhV3LkY78SPH8BUhc88rXfn2jYgN9BfiSjbcvM= +github.com/coinbase/rosetta-sdk-go v0.5.9-0.20201030213420-e8c5df9fe202 h1:tSzJ9tXxfa9UsXygm0HQL3SMhL2MTamT9nPKVb+6LG4= +github.com/coinbase/rosetta-sdk-go v0.5.9-0.20201030213420-e8c5df9fe202/go.mod h1:xd4wYUhV3LkY78SPH8BUhc88rXfn2jYgN9BfiSjbcvM= github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= From 25a2421477569d9547f7a2ed0273c4326c454fff Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Fri, 30 Oct 2020 14:37:05 -0700 Subject: [PATCH 172/376] Update version --- cmd/root.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/root.go b/cmd/root.go index d89c144f..4ea78f21 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -257,6 +257,6 @@ var versionCmd = &cobra.Command{ Use: "version", Short: "Print rosetta-cli version", Run: func(cmd *cobra.Command, args []string) { - fmt.Println("v0.5.19") + fmt.Println("v0.5.20") }, } From 43f31d6b8c52e228be37425e4e6035f4a475bb28 Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Fri, 30 Oct 2020 14:43:07 -0700 Subject: [PATCH 173/376] Fix breaking changes --- pkg/processor/reconciler_helper.go | 15 +++++++++++++++ pkg/tester/data.go | 2 ++ 2 files changed, 17 insertions(+) diff --git a/pkg/processor/reconciler_helper.go b/pkg/processor/reconciler_helper.go index ddcee94f..e1142452 100644 --- a/pkg/processor/reconciler_helper.go +++ b/pkg/processor/reconciler_helper.go @@ -35,6 +35,7 @@ type ReconcilerHelper struct { database storage.Database blockStorage *storage.BlockStorage balanceStorage *storage.BalanceStorage + tipDelay int64 } // NewReconcilerHelper returns a new ReconcilerHelper. @@ -44,6 +45,7 @@ func NewReconcilerHelper( database storage.Database, blockStorage *storage.BlockStorage, balanceStorage *storage.BalanceStorage, + tipDelay int64, ) *ReconcilerHelper { return &ReconcilerHelper{ network: network, @@ -51,6 +53,7 @@ func NewReconcilerHelper( database: database, blockStorage: blockStorage, balanceStorage: balanceStorage, + tipDelay: tipDelay, } } @@ -73,6 +76,18 @@ func (h *ReconcilerHelper) CanonicalBlock( return h.blockStorage.CanonicalBlockTransactional(ctx, block, dbTx) } +// IndexAtTip returns a boolean indicating if a block +// index is at tip (provided some acceptable +// tip delay). If the index is ahead of the head block +// and the head block is at tip, we consider the +// index at tip. +func (h *ReconcilerHelper) IndexAtTip( + ctx context.Context, + index int64, +) (bool, error) { + return h.blockStorage.IndexAtTip(ctx, h.tipDelay, index) +} + // CurrentBlock returns the last processed block and is used // to determine which block to check account balances at during // inactive reconciliation. diff --git a/pkg/tester/data.go b/pkg/tester/data.go index be3983d0..0526617e 100644 --- a/pkg/tester/data.go +++ b/pkg/tester/data.go @@ -211,6 +211,7 @@ func InitializeData( localStore, blockStorage, balanceStorage, + config.TipDelay, ) reconcilerHandler := processor.NewReconcilerHandler( @@ -954,6 +955,7 @@ func (t *DataTester) recursiveOpSearch( localStore, blockStorage, balanceStorage, + t.config.TipDelay, ) reconcilerHandler := processor.NewReconcilerHandler( From 1f341285ec1ef0d310bebdfdd6d5d185e0937cc8 Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Fri, 30 Oct 2020 14:47:38 -0700 Subject: [PATCH 174/376] Log block hashes when log blocks is enabled --- pkg/logger/logger.go | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/pkg/logger/logger.go b/pkg/logger/logger.go index f614271b..5830ff6a 100644 --- a/pkg/logger/logger.go +++ b/pkg/logger/logger.go @@ -196,15 +196,16 @@ func (l *Logger) AddBlockStream( defer closeFile(f) - _, err = f.WriteString(fmt.Sprintf( + blockString := fmt.Sprintf( "%s Block %d:%s with Parent Block %d:%s\n", addEvent, block.BlockIdentifier.Index, block.BlockIdentifier.Hash, block.ParentBlockIdentifier.Index, block.ParentBlockIdentifier.Hash, - )) - if err != nil { + ) + fmt.Printf(blockString) + if _, err := f.WriteString(blockString); err != nil { return err } @@ -232,17 +233,15 @@ func (l *Logger) RemoveBlockStream( defer closeFile(f) - _, err = f.WriteString(fmt.Sprintf( + blockString := fmt.Sprintf( "%s Block %d:%s\n", removeEvent, block.Index, block.Hash, - )) - if err != nil { - return err - } - - return nil + ) + fmt.Printf(blockString) + _, err = f.WriteString(blockString) + return err } // TransactionStream writes the next processed block's transactions From 842c846ac4106bb72be0f7c40830d80fb6b6ca65 Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Fri, 30 Oct 2020 14:54:39 -0700 Subject: [PATCH 175/376] nits --- configuration/configuration.go | 10 ++++++++-- go.sum | 3 +-- pkg/logger/logger.go | 5 ++++- pkg/results/data_results.go | 35 ++++++++++++++++++++++++++-------- pkg/tester/construction.go | 11 +++++++++-- pkg/tester/data.go | 9 +++++++-- 6 files changed, 56 insertions(+), 17 deletions(-) diff --git a/configuration/configuration.go b/configuration/configuration.go index 0908f738..c227dead 100644 --- a/configuration/configuration.go +++ b/configuration/configuration.go @@ -250,7 +250,10 @@ func assertDataConfiguration(config *DataConfiguration) error { accountCount := config.EndConditions.ReconciliationCoverage.AccountCount if accountCount != nil && *accountCount < 0 { - return fmt.Errorf("reconciliation coverage account count %d must be >= 0", *accountCount) + return fmt.Errorf( + "reconciliation coverage account count %d must be >= 0", + *accountCount, + ) } if config.BalanceTrackingDisabled { @@ -308,7 +311,10 @@ func modifyFilePaths(config *Configuration, fileDir string) { if config.Construction != nil { if len(config.Construction.ConstructorDSLFile) > 0 { - config.Construction.ConstructorDSLFile = path.Join(fileDir, config.Construction.ConstructorDSLFile) + config.Construction.ConstructorDSLFile = path.Join( + fileDir, + config.Construction.ConstructorDSLFile, + ) } } } diff --git a/go.sum b/go.sum index 219dc29f..57a001fc 100644 --- a/go.sum +++ b/go.sum @@ -76,8 +76,6 @@ github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+ github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cloudflare/cloudflare-go v0.10.2-0.20190916151808-a80f83b9add9/go.mod h1:1MxXX1Ux4x6mqPmjkUgTP1CdXIBXKX7T+Jk9Gxrmx+U= -github.com/coinbase/rosetta-sdk-go v0.5.9-0.20201029210921-d7499a34c1f6 h1:5zDhG7QoXRf2GtNeDqfnUdAe5gm1fdKy0h5Txa7NHec= -github.com/coinbase/rosetta-sdk-go v0.5.9-0.20201029210921-d7499a34c1f6/go.mod h1:xd4wYUhV3LkY78SPH8BUhc88rXfn2jYgN9BfiSjbcvM= github.com/coinbase/rosetta-sdk-go v0.5.9-0.20201030213420-e8c5df9fe202 h1:tSzJ9tXxfa9UsXygm0HQL3SMhL2MTamT9nPKVb+6LG4= github.com/coinbase/rosetta-sdk-go v0.5.9-0.20201030213420-e8c5df9fe202/go.mod h1:xd4wYUhV3LkY78SPH8BUhc88rXfn2jYgN9BfiSjbcvM= github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= @@ -513,6 +511,7 @@ golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtn golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191112195655-aa38f8e97acc/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20200117012304-6edc0a871e69 h1:yBHHx+XZqXJBm6Exke3N7V9gnlsyXxoCPEb1yVenjfk= golang.org/x/tools v0.0.0-20200117012304-6edc0a871e69/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= diff --git a/pkg/logger/logger.go b/pkg/logger/logger.go index 5830ff6a..9d6e36ec 100644 --- a/pkg/logger/logger.go +++ b/pkg/logger/logger.go @@ -143,7 +143,10 @@ func (l *Logger) LogDataStatus(ctx context.Context, status *results.CheckDataSta } // LogConstructionStatus logs results.CheckConstructionStatus. -func (l *Logger) LogConstructionStatus(ctx context.Context, status *results.CheckConstructionStatus) { +func (l *Logger) LogConstructionStatus( + ctx context.Context, + status *results.CheckConstructionStatus, +) { statsMessage := fmt.Sprintf( "[STATS] Transactions Confirmed: %d (Created: %d, In Progress: %d, Stale: %d, Failed: %d) Addresses Created: %d", status.Stats.TransactionsConfirmed, diff --git a/pkg/results/data_results.go b/pkg/results/data_results.go index bf7f5c71..950c97cc 100644 --- a/pkg/results/data_results.go +++ b/pkg/results/data_results.go @@ -334,17 +334,31 @@ func ComputeCheckDataProgress( return nil } - blocksPerSecond := new(big.Float).Quo(new(big.Float).SetInt64(adjustedBlocks), new(big.Float).SetInt(elapsedTime)) + blocksPerSecond := new( + big.Float, + ).Quo( + new(big.Float).SetInt64(adjustedBlocks), + new(big.Float).SetInt(elapsedTime), + ) blocksPerSecondFloat, _ := blocksPerSecond.Float64() - blocksSynced := new(big.Float).Quo(new(big.Float).SetInt64(headBlock.Index), new(big.Float).SetInt64(tipIndex)) + blocksSynced := new( + big.Float, + ).Quo( + new(big.Float).SetInt64(headBlock.Index), + new(big.Float).SetInt64(tipIndex), + ) blocksSyncedFloat, _ := blocksSynced.Float64() return &CheckDataProgress{ - Blocks: headBlock.Index, - Tip: tipIndex, - Completed: blocksSyncedFloat * utils.OneHundred, - Rate: blocksPerSecondFloat, - TimeRemaining: utils.TimeToTip(blocksPerSecondFloat, headBlock.Index, tipIndex).String(), + Blocks: headBlock.Index, + Tip: tipIndex, + Completed: blocksSyncedFloat * utils.OneHundred, + Rate: blocksPerSecondFloat, + TimeRemaining: utils.TimeToTip( + blocksPerSecondFloat, + headBlock.Index, + tipIndex, + ).String(), ReconcilerQueueSize: reconciler.QueueSize(), ReconcilerLastIndex: reconciler.LastIndexReconciled(), } @@ -604,7 +618,12 @@ func ComputeCheckDataTests( // nolint:gocognit ResponseAssertion: ResponseAssertionTest(err), BlockSyncing: BlockSyncingTest(err, blocksSynced), BalanceTracking: BalanceTrackingTest(cfg, err, operationsSeen), - Reconciliation: ReconciliationTest(cfg, err, reconciliationsPerformed, reconciliationsFailed), + Reconciliation: ReconciliationTest( + cfg, + err, + reconciliationsPerformed, + reconciliationsFailed, + ), } } diff --git a/pkg/tester/construction.go b/pkg/tester/construction.go index 8745fbe9..625375a9 100644 --- a/pkg/tester/construction.go +++ b/pkg/tester/construction.go @@ -102,7 +102,8 @@ func InitializeConstruction( log.Fatalf("%s: unable to get network options", fetchErr.Err.Error()) } - if len(networkOptions.Allow.BalanceExemptions) > 0 && config.Construction.InitialBalanceFetchDisabled { + if len(networkOptions.Allow.BalanceExemptions) > 0 && + config.Construction.InitialBalanceFetchDisabled { log.Fatal("found balance exemptions but initial balance fetch disabled") } @@ -296,7 +297,13 @@ func (t *ConstructionTester) StartPeriodicLogger( case <-ctx.Done(): return ctx.Err() case <-tc.C: - status := results.ComputeCheckConstructionStatus(ctx, t.config, t.counterStorage, t.broadcastStorage, t.jobStorage) + status := results.ComputeCheckConstructionStatus( + ctx, + t.config, + t.counterStorage, + t.broadcastStorage, + t.jobStorage, + ) t.logger.LogConstructionStatus(ctx, status) } } diff --git a/pkg/tester/data.go b/pkg/tester/data.go index 0526617e..8a382741 100644 --- a/pkg/tester/data.go +++ b/pkg/tester/data.go @@ -733,7 +733,10 @@ func (t *DataTester) WaitForEmptyQueue( // DrainReconcilerQueue returns once the reconciler queue has been drained // or an error is encountered. -func (t *DataTester) DrainReconcilerQueue(ctx context.Context, sigListeners *[]context.CancelFunc) error { +func (t *DataTester) DrainReconcilerQueue( + ctx context.Context, + sigListeners *[]context.CancelFunc, +) error { color.Cyan("draining reconciler backlog (you can disable this in your configuration file)") // To cancel all execution, need to call multiple cancel functions. @@ -802,7 +805,9 @@ func (t *DataTester) HandleErr(err error, sigListeners *[]context.CancelFunc) er if shouldReconcile(t.config) && t.reconciler.QueueSize() > 0 { if t.config.Data.ReconciliationDrainDisabled { - color.Cyan("skipping reconciler backlog drain (you can enable this in your configuration file)") + color.Cyan( + "skipping reconciler backlog drain (you can enable this in your configuration file)", + ) } else { drainErr := t.DrainReconcilerQueue(ctx, sigListeners) if drainErr != nil { From 77e1f56b834a9ffd1d2e99178f0e0d4bd89b74be Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Fri, 30 Oct 2020 14:57:35 -0700 Subject: [PATCH 176/376] more nits --- pkg/logger/logger.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/logger/logger.go b/pkg/logger/logger.go index 9d6e36ec..07f8f11f 100644 --- a/pkg/logger/logger.go +++ b/pkg/logger/logger.go @@ -207,7 +207,7 @@ func (l *Logger) AddBlockStream( block.ParentBlockIdentifier.Index, block.ParentBlockIdentifier.Hash, ) - fmt.Printf(blockString) + fmt.Print(blockString) if _, err := f.WriteString(blockString); err != nil { return err } @@ -242,7 +242,7 @@ func (l *Logger) RemoveBlockStream( block.Index, block.Hash, ) - fmt.Printf(blockString) + fmt.Print(blockString) _, err = f.WriteString(blockString) return err } From cc154c117475596796416f711e1d02c826c06f3b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 2 Nov 2020 05:34:29 +0000 Subject: [PATCH 177/376] Bump github.com/fatih/color from 1.9.0 to 1.10.0 Bumps [github.com/fatih/color](https://github.com/fatih/color) from 1.9.0 to 1.10.0. - [Release notes](https://github.com/fatih/color/releases) - [Commits](https://github.com/fatih/color/compare/v1.9.0...v1.10.0) Signed-off-by: dependabot[bot] --- go.mod | 2 +- go.sum | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index bda12ee1..6f3004a0 100644 --- a/go.mod +++ b/go.mod @@ -4,7 +4,7 @@ go 1.13 require ( github.com/coinbase/rosetta-sdk-go v0.5.9-0.20201030213420-e8c5df9fe202 - github.com/fatih/color v1.9.0 + github.com/fatih/color v1.10.0 github.com/olekukonko/tablewriter v0.0.2-0.20190409134802-7e037d187b0c github.com/spf13/cobra v1.1.1 github.com/stretchr/testify v1.6.1 diff --git a/go.sum b/go.sum index 57a001fc..d60b2dec 100644 --- a/go.sum +++ b/go.sum @@ -121,6 +121,8 @@ github.com/fatih/color v1.3.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5Kwzbycv github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/fatih/color v1.9.0 h1:8xPHl4/q1VyqGIPif1F+1V3Y3lSmrq01EabUW3CoW5s= github.com/fatih/color v1.9.0/go.mod h1:eQcE1qtQxscV5RaZvpXrrb8Drkc3/DdQ+uUYCNjL+zU= +github.com/fatih/color v1.10.0 h1:s36xzo75JdqLaaWoiEHk767eHiwo0598uUxyfiPkDsg= +github.com/fatih/color v1.10.0/go.mod h1:ELkj/draVOlAH/xkhN6mQ50Qd0MPOk5AAr3maGEBuJM= github.com/fjl/memsize v0.0.0-20180418122429-ca190fb6ffbc/go.mod h1:VvhXpOYNQvB+uIk2RvXzuaQtkQJzzIx6lSBe1xv7hi0= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= @@ -245,6 +247,8 @@ github.com/mattn/go-colorable v0.1.4 h1:snbPLB8fVfU9iwbbo30TPtbLRzwWu6aJS6Xh4eaa github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= github.com/mattn/go-colorable v0.1.7 h1:bQGKb3vps/j0E9GfJQ03JyhRuxsvdAanXlT9BTw3mdw= github.com/mattn/go-colorable v0.1.7/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= +github.com/mattn/go-colorable v0.1.8 h1:c1ghPdyEDarC70ftn0y+A/Ee++9zz8ljHG1b13eJ0s8= +github.com/mattn/go-colorable v0.1.8/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= github.com/mattn/go-ieproxy v0.0.0-20190610004146-91bb50d98149/go.mod h1:31jz6HNzdxOmlERGGEc4v/dMssOfmp2p5bT/okiKFFc= github.com/mattn/go-ieproxy v0.0.0-20190702010315-6dee0af9227d/go.mod h1:31jz6HNzdxOmlERGGEc4v/dMssOfmp2p5bT/okiKFFc= github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= From a02d90a8f38a1931689a8b6179ccd4481caac3c4 Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Tue, 3 Nov 2020 14:02:51 -0800 Subject: [PATCH 178/376] Update rosetta-sdk-go --- go.mod | 2 +- go.sum | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index 6f3004a0..5178e249 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/coinbase/rosetta-cli go 1.13 require ( - github.com/coinbase/rosetta-sdk-go v0.5.9-0.20201030213420-e8c5df9fe202 + github.com/coinbase/rosetta-sdk-go v0.5.10-0.20201103220049-3a720608da79 github.com/fatih/color v1.10.0 github.com/olekukonko/tablewriter v0.0.2-0.20190409134802-7e037d187b0c github.com/spf13/cobra v1.1.1 diff --git a/go.sum b/go.sum index d60b2dec..2d27626d 100644 --- a/go.sum +++ b/go.sum @@ -78,6 +78,8 @@ github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDk github.com/cloudflare/cloudflare-go v0.10.2-0.20190916151808-a80f83b9add9/go.mod h1:1MxXX1Ux4x6mqPmjkUgTP1CdXIBXKX7T+Jk9Gxrmx+U= github.com/coinbase/rosetta-sdk-go v0.5.9-0.20201030213420-e8c5df9fe202 h1:tSzJ9tXxfa9UsXygm0HQL3SMhL2MTamT9nPKVb+6LG4= github.com/coinbase/rosetta-sdk-go v0.5.9-0.20201030213420-e8c5df9fe202/go.mod h1:xd4wYUhV3LkY78SPH8BUhc88rXfn2jYgN9BfiSjbcvM= +github.com/coinbase/rosetta-sdk-go v0.5.10-0.20201103220049-3a720608da79 h1:UvuAOEDni8pSopTNfrZHel9EEPzvPxLD8bjqMmP23cc= +github.com/coinbase/rosetta-sdk-go v0.5.10-0.20201103220049-3a720608da79/go.mod h1:pqBibzTcpz0mLqb7is8fo2qf93tcHjItJ3u7Yud8RI4= github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= From f0e14713dd0d6fdba5fb99514d95e2b3e3f94217 Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Tue, 3 Nov 2020 14:03:39 -0800 Subject: [PATCH 179/376] Update rosetta-cli version --- cmd/root.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/root.go b/cmd/root.go index 4ea78f21..61e14ca4 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -257,6 +257,6 @@ var versionCmd = &cobra.Command{ Use: "version", Short: "Print rosetta-cli version", Run: func(cmd *cobra.Command, args []string) { - fmt.Println("v0.5.20") + fmt.Println("v0.5.21") }, } From 73561cfaeed85b7498a822267b567ae3e24a8cc1 Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Wed, 4 Nov 2020 14:29:34 -0800 Subject: [PATCH 180/376] Don't prune balances if pruning disabled --- pkg/processor/reconciler_helper.go | 19 +++++++++++++++---- pkg/tester/data.go | 4 ++-- 2 files changed, 17 insertions(+), 6 deletions(-) diff --git a/pkg/processor/reconciler_helper.go b/pkg/processor/reconciler_helper.go index e1142452..b533ad6d 100644 --- a/pkg/processor/reconciler_helper.go +++ b/pkg/processor/reconciler_helper.go @@ -17,6 +17,8 @@ package processor import ( "context" + "github.com/coinbase/rosetta-cli/configuration" + "github.com/coinbase/rosetta-sdk-go/fetcher" "github.com/coinbase/rosetta-sdk-go/reconciler" "github.com/coinbase/rosetta-sdk-go/storage" @@ -29,31 +31,32 @@ var _ reconciler.Helper = (*ReconcilerHelper)(nil) // ReconcilerHelper implements the Reconciler.Helper // interface. type ReconcilerHelper struct { + config *configuration.Configuration + network *types.NetworkIdentifier fetcher *fetcher.Fetcher database storage.Database blockStorage *storage.BlockStorage balanceStorage *storage.BalanceStorage - tipDelay int64 } // NewReconcilerHelper returns a new ReconcilerHelper. func NewReconcilerHelper( + config *configuration.Configuration, network *types.NetworkIdentifier, fetcher *fetcher.Fetcher, database storage.Database, blockStorage *storage.BlockStorage, balanceStorage *storage.BalanceStorage, - tipDelay int64, ) *ReconcilerHelper { return &ReconcilerHelper{ + config: config, network: network, fetcher: fetcher, database: database, blockStorage: blockStorage, balanceStorage: balanceStorage, - tipDelay: tipDelay, } } @@ -85,7 +88,11 @@ func (h *ReconcilerHelper) IndexAtTip( ctx context.Context, index int64, ) (bool, error) { - return h.blockStorage.IndexAtTip(ctx, h.tipDelay, index) + return h.blockStorage.IndexAtTip( + ctx, + h.config.TipDelay, + index, + ) } // CurrentBlock returns the last processed block and is used @@ -142,6 +149,10 @@ func (h *ReconcilerHelper) PruneBalances( currency *types.Currency, index int64, ) error { + if h.config.Data.PruningDisabled { + return nil + } + return h.balanceStorage.PruneBalances( ctx, account, diff --git a/pkg/tester/data.go b/pkg/tester/data.go index 8a382741..f8df23d2 100644 --- a/pkg/tester/data.go +++ b/pkg/tester/data.go @@ -206,12 +206,12 @@ func InitializeData( ) reconcilerHelper := processor.NewReconcilerHelper( + config, network, fetcher, localStore, blockStorage, balanceStorage, - config.TipDelay, ) reconcilerHandler := processor.NewReconcilerHandler( @@ -955,12 +955,12 @@ func (t *DataTester) recursiveOpSearch( ) reconcilerHelper := processor.NewReconcilerHelper( + t.config, t.network, t.fetcher, localStore, blockStorage, balanceStorage, - t.config.TipDelay, ) reconcilerHandler := processor.NewReconcilerHandler( From bf04f2034151b3b4c6d98b6100d202b7e37e9274 Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Wed, 4 Nov 2020 14:32:35 -0800 Subject: [PATCH 181/376] Increase rosetta-cli version --- cmd/root.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/root.go b/cmd/root.go index 61e14ca4..a6bfcfee 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -257,6 +257,6 @@ var versionCmd = &cobra.Command{ Use: "version", Short: "Print rosetta-cli version", Run: func(cmd *cobra.Command, args []string) { - fmt.Println("v0.5.21") + fmt.Println("v0.5.22") }, } From d2ee85cffff89d2688882cbb245a0a76834eeaa8 Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Wed, 4 Nov 2020 19:19:00 -0800 Subject: [PATCH 182/376] Update rosetta-sdk-go --- go.mod | 2 +- go.sum | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index 5178e249..541e6d8a 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/coinbase/rosetta-cli go 1.13 require ( - github.com/coinbase/rosetta-sdk-go v0.5.10-0.20201103220049-3a720608da79 + github.com/coinbase/rosetta-sdk-go v0.5.10-0.20201105031224-eeb3d969685f github.com/fatih/color v1.10.0 github.com/olekukonko/tablewriter v0.0.2-0.20190409134802-7e037d187b0c github.com/spf13/cobra v1.1.1 diff --git a/go.sum b/go.sum index 2d27626d..4fe50310 100644 --- a/go.sum +++ b/go.sum @@ -80,6 +80,8 @@ github.com/coinbase/rosetta-sdk-go v0.5.9-0.20201030213420-e8c5df9fe202 h1:tSzJ9 github.com/coinbase/rosetta-sdk-go v0.5.9-0.20201030213420-e8c5df9fe202/go.mod h1:xd4wYUhV3LkY78SPH8BUhc88rXfn2jYgN9BfiSjbcvM= github.com/coinbase/rosetta-sdk-go v0.5.10-0.20201103220049-3a720608da79 h1:UvuAOEDni8pSopTNfrZHel9EEPzvPxLD8bjqMmP23cc= github.com/coinbase/rosetta-sdk-go v0.5.10-0.20201103220049-3a720608da79/go.mod h1:pqBibzTcpz0mLqb7is8fo2qf93tcHjItJ3u7Yud8RI4= +github.com/coinbase/rosetta-sdk-go v0.5.10-0.20201105031224-eeb3d969685f h1:/w0ECMQ4kpd0vPitSOzkKteGSxmbxPACcJEbQHMfEf8= +github.com/coinbase/rosetta-sdk-go v0.5.10-0.20201105031224-eeb3d969685f/go.mod h1:pqBibzTcpz0mLqb7is8fo2qf93tcHjItJ3u7Yud8RI4= github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= From f4ac27dc48cda048ec4d27e388b8de3058ac9095 Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Wed, 4 Nov 2020 19:31:31 -0800 Subject: [PATCH 183/376] Add MaxReorgDepth to config file --- configuration/configuration.go | 4 ++++ configuration/types.go | 10 ++++++++++ 2 files changed, 14 insertions(+) diff --git a/configuration/configuration.go b/configuration/configuration.go index c227dead..d7e48c20 100644 --- a/configuration/configuration.go +++ b/configuration/configuration.go @@ -148,6 +148,10 @@ func populateMissingFields(config *Configuration) *Configuration { config.TipDelay = DefaultTipDelay } + if config.MaxReorgDepth == 0 { + config.MaxReorgDepth = DefaultMaxReorgDepth + } + config.Construction = populateConstructionMissingFields(config.Construction) config.Data = populateDataMissingFields(config.Data) diff --git a/configuration/types.go b/configuration/types.go index 9a6a51d8..d04cc643 100644 --- a/configuration/types.go +++ b/configuration/types.go @@ -59,6 +59,7 @@ const ( DefaultTipDelay = 300 DefaultBlockBroadcastLimit = 5 DefaultStatusPort = 9090 + DefaultMaxReorgDepth = 100 // ETH Defaults EthereumIDBlockchain = "Ethereum" @@ -366,6 +367,15 @@ type Configuration struct { // we are considered to be behind tip. TipDelay int64 `json:"tip_delay"` + // MaxReorgDepth specifies the maximum possible reorg depth of the blockchain + // being synced. This value is used to determine how aggressively to prune + // old block data. + // + // It is better to be overly cautious here as keeping a few + // too many blocks around is much better than running into an + // error caused by missing block data! + MaxReorgDepth int64 `json:"max_reorg_depth,omitempty"` + // LogConfiguration determines if the configuration settings // should be printed to the console when a file is loaded. LogConfiguration bool `json:"log_configuration"` From 7288f55bf475e5b9a11a63e3694243bcfaf4a12e Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Wed, 4 Nov 2020 19:34:42 -0800 Subject: [PATCH 184/376] Use MaxReorgDepth in rosetta-cli init --- configuration/types.go | 2 +- pkg/tester/construction.go | 1 + pkg/tester/data.go | 4 +++- 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/configuration/types.go b/configuration/types.go index d04cc643..8feb3837 100644 --- a/configuration/types.go +++ b/configuration/types.go @@ -374,7 +374,7 @@ type Configuration struct { // It is better to be overly cautious here as keeping a few // too many blocks around is much better than running into an // error caused by missing block data! - MaxReorgDepth int64 `json:"max_reorg_depth,omitempty"` + MaxReorgDepth int `json:"max_reorg_depth,omitempty"` // LogConfiguration determines if the configuration settings // should be printed to the console when a file is loaded. diff --git a/pkg/tester/construction.go b/pkg/tester/construction.go index 625375a9..194198ae 100644 --- a/pkg/tester/construction.go +++ b/pkg/tester/construction.go @@ -258,6 +258,7 @@ func InitializeConstruction( []storage.BlockWorker{balanceStorage, coinStorage, broadcastStorage}, syncer.DefaultCacheSize, config.MaxSyncConcurrency, + config.MaxReorgDepth, ) return &ConstructionTester{ diff --git a/pkg/tester/data.go b/pkg/tester/data.go index f8df23d2..068b30ba 100644 --- a/pkg/tester/data.go +++ b/pkg/tester/data.go @@ -317,6 +317,7 @@ func InitializeData( blockWorkers, syncer.DefaultCacheSize, config.MaxSyncConcurrency, + config.MaxReorgDepth, ) return &DataTester{ @@ -381,7 +382,7 @@ func (t *DataTester) PruneableIndex( // balances at their index. // // It is ok if the returned value here is negative. - return headIndex - statefulsyncer.DefaultPruningDepth, nil + return headIndex - int64(t.config.MaxReorgDepth), nil } // StartReconciler starts the reconciler if @@ -1017,6 +1018,7 @@ func (t *DataTester) recursiveOpSearch( []storage.BlockWorker{balanceStorage}, syncer.DefaultCacheSize, t.config.MaxSyncConcurrency, + t.config.MaxReorgDepth, ) g, ctx := errgroup.WithContext(ctx) From 60d695628f7c235329d95ce4cdadd4cb8307ed5b Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Wed, 4 Nov 2020 20:11:57 -0800 Subject: [PATCH 185/376] don't prune during inactive reconciliation --- go.mod | 2 +- go.sum | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index 541e6d8a..1da5a95e 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/coinbase/rosetta-cli go 1.13 require ( - github.com/coinbase/rosetta-sdk-go v0.5.10-0.20201105031224-eeb3d969685f + github.com/coinbase/rosetta-sdk-go v0.5.10-0.20201105041044-671e2d01699e github.com/fatih/color v1.10.0 github.com/olekukonko/tablewriter v0.0.2-0.20190409134802-7e037d187b0c github.com/spf13/cobra v1.1.1 diff --git a/go.sum b/go.sum index 4fe50310..812e3e5a 100644 --- a/go.sum +++ b/go.sum @@ -82,6 +82,8 @@ github.com/coinbase/rosetta-sdk-go v0.5.10-0.20201103220049-3a720608da79 h1:UvuA github.com/coinbase/rosetta-sdk-go v0.5.10-0.20201103220049-3a720608da79/go.mod h1:pqBibzTcpz0mLqb7is8fo2qf93tcHjItJ3u7Yud8RI4= github.com/coinbase/rosetta-sdk-go v0.5.10-0.20201105031224-eeb3d969685f h1:/w0ECMQ4kpd0vPitSOzkKteGSxmbxPACcJEbQHMfEf8= github.com/coinbase/rosetta-sdk-go v0.5.10-0.20201105031224-eeb3d969685f/go.mod h1:pqBibzTcpz0mLqb7is8fo2qf93tcHjItJ3u7Yud8RI4= +github.com/coinbase/rosetta-sdk-go v0.5.10-0.20201105041044-671e2d01699e h1:g66k0AI2GvNuop+khjY0qO72l7JIO6ma1QbH6fJLnn8= +github.com/coinbase/rosetta-sdk-go v0.5.10-0.20201105041044-671e2d01699e/go.mod h1:pqBibzTcpz0mLqb7is8fo2qf93tcHjItJ3u7Yud8RI4= github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= From d5705c80c8d26b1019f381a344b061f58200ea53 Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Wed, 4 Nov 2020 20:17:29 -0800 Subject: [PATCH 186/376] Fix configuration tests --- configuration/configuration.go | 1 + configuration/configuration_test.go | 1 + 2 files changed, 2 insertions(+) diff --git a/configuration/configuration.go b/configuration/configuration.go index d7e48c20..1f6df0dd 100644 --- a/configuration/configuration.go +++ b/configuration/configuration.go @@ -53,6 +53,7 @@ func DefaultConfiguration() *Configuration { MaxRetries: DefaultMaxRetries, MaxSyncConcurrency: DefaultMaxSyncConcurrency, TipDelay: DefaultTipDelay, + MaxReorgDepth: DefaultMaxReorgDepth, Data: DefaultDataConfiguration(), } } diff --git a/configuration/configuration_test.go b/configuration/configuration_test.go index 84610f55..7cc2c503 100644 --- a/configuration/configuration_test.go +++ b/configuration/configuration_test.go @@ -67,6 +67,7 @@ var ( MaxRetries: 1000, MaxSyncConcurrency: 12, TipDelay: 1231, + MaxReorgDepth: 12, Construction: &ConstructionConfiguration{ OfflineURL: "https://ashdjaksdkjshdk", MaxOfflineConnections: 21, From 21596920cc02d6b0b0c0ce91e79c0a552df21794 Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Wed, 4 Nov 2020 20:21:45 -0800 Subject: [PATCH 187/376] Update example config --- examples/configuration/default.json | 1 + 1 file changed, 1 insertion(+) diff --git a/examples/configuration/default.json b/examples/configuration/default.json index 4fe1a204..1d5a448a 100644 --- a/examples/configuration/default.json +++ b/examples/configuration/default.json @@ -11,6 +11,7 @@ "max_online_connections": 120, "max_sync_concurrency": 64, "tip_delay": 300, + "max_reorg_depth": 100, "log_configuration": false, "compression_disabled": false, "memory_limit_disabled": false, From 40b23fb17e0bed5a90b8aefed62aad0b3314096f Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Wed, 4 Nov 2020 20:44:18 -0800 Subject: [PATCH 188/376] update rosetta-sdk-go to tip --- go.mod | 2 +- go.sum | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index 1da5a95e..a0f910e4 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/coinbase/rosetta-cli go 1.13 require ( - github.com/coinbase/rosetta-sdk-go v0.5.10-0.20201105041044-671e2d01699e + github.com/coinbase/rosetta-sdk-go v0.5.10-0.20201105044327-5d14e78bb3c7 github.com/fatih/color v1.10.0 github.com/olekukonko/tablewriter v0.0.2-0.20190409134802-7e037d187b0c github.com/spf13/cobra v1.1.1 diff --git a/go.sum b/go.sum index 812e3e5a..b740f0f1 100644 --- a/go.sum +++ b/go.sum @@ -84,6 +84,8 @@ github.com/coinbase/rosetta-sdk-go v0.5.10-0.20201105031224-eeb3d969685f h1:/w0E github.com/coinbase/rosetta-sdk-go v0.5.10-0.20201105031224-eeb3d969685f/go.mod h1:pqBibzTcpz0mLqb7is8fo2qf93tcHjItJ3u7Yud8RI4= github.com/coinbase/rosetta-sdk-go v0.5.10-0.20201105041044-671e2d01699e h1:g66k0AI2GvNuop+khjY0qO72l7JIO6ma1QbH6fJLnn8= github.com/coinbase/rosetta-sdk-go v0.5.10-0.20201105041044-671e2d01699e/go.mod h1:pqBibzTcpz0mLqb7is8fo2qf93tcHjItJ3u7Yud8RI4= +github.com/coinbase/rosetta-sdk-go v0.5.10-0.20201105044327-5d14e78bb3c7 h1:Y6Kk2bar8EVLyMLNOQ76h7OSeP4+XJx6GHhn11YDwew= +github.com/coinbase/rosetta-sdk-go v0.5.10-0.20201105044327-5d14e78bb3c7/go.mod h1:pqBibzTcpz0mLqb7is8fo2qf93tcHjItJ3u7Yud8RI4= github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= From eecee5378693d05848016806f7e4cb6e41f26014 Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Thu, 5 Nov 2020 09:10:02 -0800 Subject: [PATCH 189/376] Update rosetta-sdk-go --- go.mod | 2 +- go.sum | 23 ++--------------------- 2 files changed, 3 insertions(+), 22 deletions(-) diff --git a/go.mod b/go.mod index a0f910e4..056fd68d 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/coinbase/rosetta-cli go 1.13 require ( - github.com/coinbase/rosetta-sdk-go v0.5.10-0.20201105044327-5d14e78bb3c7 + github.com/coinbase/rosetta-sdk-go v0.5.10 github.com/fatih/color v1.10.0 github.com/olekukonko/tablewriter v0.0.2-0.20190409134802-7e037d187b0c github.com/spf13/cobra v1.1.1 diff --git a/go.sum b/go.sum index b740f0f1..d3bd7611 100644 --- a/go.sum +++ b/go.sum @@ -76,16 +76,8 @@ github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+ github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cloudflare/cloudflare-go v0.10.2-0.20190916151808-a80f83b9add9/go.mod h1:1MxXX1Ux4x6mqPmjkUgTP1CdXIBXKX7T+Jk9Gxrmx+U= -github.com/coinbase/rosetta-sdk-go v0.5.9-0.20201030213420-e8c5df9fe202 h1:tSzJ9tXxfa9UsXygm0HQL3SMhL2MTamT9nPKVb+6LG4= -github.com/coinbase/rosetta-sdk-go v0.5.9-0.20201030213420-e8c5df9fe202/go.mod h1:xd4wYUhV3LkY78SPH8BUhc88rXfn2jYgN9BfiSjbcvM= -github.com/coinbase/rosetta-sdk-go v0.5.10-0.20201103220049-3a720608da79 h1:UvuAOEDni8pSopTNfrZHel9EEPzvPxLD8bjqMmP23cc= -github.com/coinbase/rosetta-sdk-go v0.5.10-0.20201103220049-3a720608da79/go.mod h1:pqBibzTcpz0mLqb7is8fo2qf93tcHjItJ3u7Yud8RI4= -github.com/coinbase/rosetta-sdk-go v0.5.10-0.20201105031224-eeb3d969685f h1:/w0ECMQ4kpd0vPitSOzkKteGSxmbxPACcJEbQHMfEf8= -github.com/coinbase/rosetta-sdk-go v0.5.10-0.20201105031224-eeb3d969685f/go.mod h1:pqBibzTcpz0mLqb7is8fo2qf93tcHjItJ3u7Yud8RI4= -github.com/coinbase/rosetta-sdk-go v0.5.10-0.20201105041044-671e2d01699e h1:g66k0AI2GvNuop+khjY0qO72l7JIO6ma1QbH6fJLnn8= -github.com/coinbase/rosetta-sdk-go v0.5.10-0.20201105041044-671e2d01699e/go.mod h1:pqBibzTcpz0mLqb7is8fo2qf93tcHjItJ3u7Yud8RI4= -github.com/coinbase/rosetta-sdk-go v0.5.10-0.20201105044327-5d14e78bb3c7 h1:Y6Kk2bar8EVLyMLNOQ76h7OSeP4+XJx6GHhn11YDwew= -github.com/coinbase/rosetta-sdk-go v0.5.10-0.20201105044327-5d14e78bb3c7/go.mod h1:pqBibzTcpz0mLqb7is8fo2qf93tcHjItJ3u7Yud8RI4= +github.com/coinbase/rosetta-sdk-go v0.5.10 h1:lXXlaxshpAWgp85TpBr8pdAL/mcvUjce2qt231yBIIg= +github.com/coinbase/rosetta-sdk-go v0.5.10/go.mod h1:pqBibzTcpz0mLqb7is8fo2qf93tcHjItJ3u7Yud8RI4= github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= @@ -127,8 +119,6 @@ github.com/ethereum/go-ethereum v1.9.23 h1:SIKhg/z4Q7AbvqcxuPYvMxf36che/Rq/Pp0Id github.com/ethereum/go-ethereum v1.9.23/go.mod h1:JIfVb6esrqALTExdz9hRYvrP0xBDf6wCncIu1hNwHpM= github.com/fatih/color v1.3.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= -github.com/fatih/color v1.9.0 h1:8xPHl4/q1VyqGIPif1F+1V3Y3lSmrq01EabUW3CoW5s= -github.com/fatih/color v1.9.0/go.mod h1:eQcE1qtQxscV5RaZvpXrrb8Drkc3/DdQ+uUYCNjL+zU= github.com/fatih/color v1.10.0 h1:s36xzo75JdqLaaWoiEHk767eHiwo0598uUxyfiPkDsg= github.com/fatih/color v1.10.0/go.mod h1:ELkj/draVOlAH/xkhN6mQ50Qd0MPOk5AAr3maGEBuJM= github.com/fjl/memsize v0.0.0-20180418122429-ca190fb6ffbc/go.mod h1:VvhXpOYNQvB+uIk2RvXzuaQtkQJzzIx6lSBe1xv7hi0= @@ -251,19 +241,12 @@ github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czP github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= github.com/mattn/go-colorable v0.1.0/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= -github.com/mattn/go-colorable v0.1.4 h1:snbPLB8fVfU9iwbbo30TPtbLRzwWu6aJS6Xh4eaaviA= -github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= -github.com/mattn/go-colorable v0.1.7 h1:bQGKb3vps/j0E9GfJQ03JyhRuxsvdAanXlT9BTw3mdw= -github.com/mattn/go-colorable v0.1.7/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= github.com/mattn/go-colorable v0.1.8 h1:c1ghPdyEDarC70ftn0y+A/Ee++9zz8ljHG1b13eJ0s8= github.com/mattn/go-colorable v0.1.8/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= github.com/mattn/go-ieproxy v0.0.0-20190610004146-91bb50d98149/go.mod h1:31jz6HNzdxOmlERGGEc4v/dMssOfmp2p5bT/okiKFFc= github.com/mattn/go-ieproxy v0.0.0-20190702010315-6dee0af9227d/go.mod h1:31jz6HNzdxOmlERGGEc4v/dMssOfmp2p5bT/okiKFFc= github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-isatty v0.0.5-0.20180830101745-3fb116b82035/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= -github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= -github.com/mattn/go-isatty v0.0.11 h1:FxPOTFNqGkuDUGi3H/qkUbQO4ZiBa2brKq5r0l8TGeM= -github.com/mattn/go-isatty v0.0.11/go.mod h1:PhnuNfih5lzO57/f3n+odYbM4JtupLOxQOAqxQCu2WE= github.com/mattn/go-isatty v0.0.12 h1:wuysRhFDzyxgEmMf5xjvJ2M9dZoWAXNNr5LSBS7uHXY= github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= github.com/mattn/go-runewidth v0.0.3/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= @@ -477,7 +460,6 @@ golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5h golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -488,7 +470,6 @@ golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb h1:fgwFCsaw9buMuxNd6+DQfAuSF golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= From 256b8268915c996132f3390c8ec38f3c98ca47da Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Fri, 6 Nov 2020 14:04:31 -0800 Subject: [PATCH 190/376] Handle 502, 503, and 504 --- cmd/root.go | 2 +- go.mod | 2 +- go.sum | 2 ++ 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/cmd/root.go b/cmd/root.go index a6bfcfee..76fbc6e8 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -257,6 +257,6 @@ var versionCmd = &cobra.Command{ Use: "version", Short: "Print rosetta-cli version", Run: func(cmd *cobra.Command, args []string) { - fmt.Println("v0.5.22") + fmt.Println("v0.5.23") }, } diff --git a/go.mod b/go.mod index 056fd68d..a551b88f 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/coinbase/rosetta-cli go 1.13 require ( - github.com/coinbase/rosetta-sdk-go v0.5.10 + github.com/coinbase/rosetta-sdk-go v0.5.11-0.20201106052603-0322f7b18be8 github.com/fatih/color v1.10.0 github.com/olekukonko/tablewriter v0.0.2-0.20190409134802-7e037d187b0c github.com/spf13/cobra v1.1.1 diff --git a/go.sum b/go.sum index d3bd7611..67c0b075 100644 --- a/go.sum +++ b/go.sum @@ -78,6 +78,8 @@ github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDk github.com/cloudflare/cloudflare-go v0.10.2-0.20190916151808-a80f83b9add9/go.mod h1:1MxXX1Ux4x6mqPmjkUgTP1CdXIBXKX7T+Jk9Gxrmx+U= github.com/coinbase/rosetta-sdk-go v0.5.10 h1:lXXlaxshpAWgp85TpBr8pdAL/mcvUjce2qt231yBIIg= github.com/coinbase/rosetta-sdk-go v0.5.10/go.mod h1:pqBibzTcpz0mLqb7is8fo2qf93tcHjItJ3u7Yud8RI4= +github.com/coinbase/rosetta-sdk-go v0.5.11-0.20201106052603-0322f7b18be8 h1:rIPzKk6a2adG0VqC5Rh0fipPlQbaf41yiKRriAIT3gc= +github.com/coinbase/rosetta-sdk-go v0.5.11-0.20201106052603-0322f7b18be8/go.mod h1:pqBibzTcpz0mLqb7is8fo2qf93tcHjItJ3u7Yud8RI4= github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= From b5beaab0734963f8b2e3b3f79949160283ba385a Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Mon, 9 Nov 2020 20:38:50 -0800 Subject: [PATCH 191/376] Update rosetta-sdk-go --- go.mod | 2 +- go.sum | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index a551b88f..45f9ec1c 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/coinbase/rosetta-cli go 1.13 require ( - github.com/coinbase/rosetta-sdk-go v0.5.11-0.20201106052603-0322f7b18be8 + github.com/coinbase/rosetta-sdk-go v0.5.11-0.20201110043432-d86a15694f49 github.com/fatih/color v1.10.0 github.com/olekukonko/tablewriter v0.0.2-0.20190409134802-7e037d187b0c github.com/spf13/cobra v1.1.1 diff --git a/go.sum b/go.sum index 67c0b075..21d1731f 100644 --- a/go.sum +++ b/go.sum @@ -80,6 +80,8 @@ github.com/coinbase/rosetta-sdk-go v0.5.10 h1:lXXlaxshpAWgp85TpBr8pdAL/mcvUjce2q github.com/coinbase/rosetta-sdk-go v0.5.10/go.mod h1:pqBibzTcpz0mLqb7is8fo2qf93tcHjItJ3u7Yud8RI4= github.com/coinbase/rosetta-sdk-go v0.5.11-0.20201106052603-0322f7b18be8 h1:rIPzKk6a2adG0VqC5Rh0fipPlQbaf41yiKRriAIT3gc= github.com/coinbase/rosetta-sdk-go v0.5.11-0.20201106052603-0322f7b18be8/go.mod h1:pqBibzTcpz0mLqb7is8fo2qf93tcHjItJ3u7Yud8RI4= +github.com/coinbase/rosetta-sdk-go v0.5.11-0.20201110043432-d86a15694f49 h1:fYkVRB7RqtesVO1azgT3s1RB4bDfmPlfCdmW+lFwb9A= +github.com/coinbase/rosetta-sdk-go v0.5.11-0.20201110043432-d86a15694f49/go.mod h1:pqBibzTcpz0mLqb7is8fo2qf93tcHjItJ3u7Yud8RI4= github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= From 5e9c6946728cf54cedaca40c4a47be26feeda692 Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Mon, 9 Nov 2020 21:03:58 -0800 Subject: [PATCH 192/376] Fix breaking changes --- cmd/{view_account.go => view_balance.go} | 12 ++++++------ go.mod | 2 +- go.sum | 8 ++------ pkg/logger/logger.go | 2 +- pkg/processor/balance_storage_helper.go | 2 +- pkg/processor/reconciler_helper.go | 2 +- 6 files changed, 12 insertions(+), 16 deletions(-) rename cmd/{view_account.go => view_balance.go} (90%) diff --git a/cmd/view_account.go b/cmd/view_balance.go similarity index 90% rename from cmd/view_account.go rename to cmd/view_balance.go index e4d33ff6..61ca8cb0 100644 --- a/cmd/view_account.go +++ b/cmd/view_balance.go @@ -30,22 +30,22 @@ import ( var ( viewAccountCmd = &cobra.Command{ - Use: "view:account", + Use: "view:balance", Short: "View an account balance", Long: `While debugging, it is often useful to inspect the state of an account at a certain block. This command allows you to look up any account by providing a JSON representation of a types.AccountIdentifier (and optionally a height to perform the query). -For example, you could run view:account '{"address":"interesting address"}' 1000 +For example, you could run view:balance '{"address":"interesting address"}' 1000 to lookup the balance of an interesting address at block 1000. Allowing the address to specified as JSON allows for querying by SubAccountIdentifier.`, - RunE: runViewAccountCmd, + RunE: runViewBalanceCmd, Args: cobra.MinimumNArgs(1), } ) -func runViewAccountCmd(cmd *cobra.Command, args []string) error { +func runViewBalanceCmd(cmd *cobra.Command, args []string) error { account := &types.AccountIdentifier{} if err := json.Unmarshal([]byte(args[0]), account); err != nil { return fmt.Errorf("%w: unable to unmarshal account %s", err, args[0]) @@ -84,18 +84,18 @@ func runViewAccountCmd(cmd *cobra.Command, args []string) error { lookupBlock = &types.PartialBlockIdentifier{Index: &index} } - block, amounts, coins, metadata, fetchErr := newFetcher.AccountBalanceRetry( + block, amounts, metadata, fetchErr := newFetcher.AccountBalanceRetry( Context, Config.Network, account, lookupBlock, + nil, ) if fetchErr != nil { return fmt.Errorf("%w: unable to fetch account %+v", fetchErr.Err, account) } log.Printf("Amounts: %s\n", types.PrettyPrintStruct(amounts)) - log.Printf("Coins: %s\n", types.PrettyPrintStruct(coins)) log.Printf("Metadata: %s\n", types.PrettyPrintStruct(metadata)) log.Printf("Balance Fetched At: %s\n", types.PrettyPrintStruct(block)) diff --git a/go.mod b/go.mod index 45f9ec1c..e51f03d0 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/coinbase/rosetta-cli go 1.13 require ( - github.com/coinbase/rosetta-sdk-go v0.5.11-0.20201110043432-d86a15694f49 + github.com/coinbase/rosetta-sdk-go v0.5.11-0.20201110050224-e6be00ef9da1 github.com/fatih/color v1.10.0 github.com/olekukonko/tablewriter v0.0.2-0.20190409134802-7e037d187b0c github.com/spf13/cobra v1.1.1 diff --git a/go.sum b/go.sum index 21d1731f..82da2c2a 100644 --- a/go.sum +++ b/go.sum @@ -76,12 +76,8 @@ github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+ github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cloudflare/cloudflare-go v0.10.2-0.20190916151808-a80f83b9add9/go.mod h1:1MxXX1Ux4x6mqPmjkUgTP1CdXIBXKX7T+Jk9Gxrmx+U= -github.com/coinbase/rosetta-sdk-go v0.5.10 h1:lXXlaxshpAWgp85TpBr8pdAL/mcvUjce2qt231yBIIg= -github.com/coinbase/rosetta-sdk-go v0.5.10/go.mod h1:pqBibzTcpz0mLqb7is8fo2qf93tcHjItJ3u7Yud8RI4= -github.com/coinbase/rosetta-sdk-go v0.5.11-0.20201106052603-0322f7b18be8 h1:rIPzKk6a2adG0VqC5Rh0fipPlQbaf41yiKRriAIT3gc= -github.com/coinbase/rosetta-sdk-go v0.5.11-0.20201106052603-0322f7b18be8/go.mod h1:pqBibzTcpz0mLqb7is8fo2qf93tcHjItJ3u7Yud8RI4= -github.com/coinbase/rosetta-sdk-go v0.5.11-0.20201110043432-d86a15694f49 h1:fYkVRB7RqtesVO1azgT3s1RB4bDfmPlfCdmW+lFwb9A= -github.com/coinbase/rosetta-sdk-go v0.5.11-0.20201110043432-d86a15694f49/go.mod h1:pqBibzTcpz0mLqb7is8fo2qf93tcHjItJ3u7Yud8RI4= +github.com/coinbase/rosetta-sdk-go v0.5.11-0.20201110050224-e6be00ef9da1 h1:CAceh+0T+53aGGsEf+AKJVxtIaU3pUhEqDN8HrnLXS4= +github.com/coinbase/rosetta-sdk-go v0.5.11-0.20201110050224-e6be00ef9da1/go.mod h1:pqBibzTcpz0mLqb7is8fo2qf93tcHjItJ3u7Yud8RI4= github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= diff --git a/pkg/logger/logger.go b/pkg/logger/logger.go index 07f8f11f..eda0378f 100644 --- a/pkg/logger/logger.go +++ b/pkg/logger/logger.go @@ -304,7 +304,7 @@ func (l *Logger) TransactionStream( participant, amount, symbol, - op.Status, + *op.Status, )) if err != nil { return err diff --git a/pkg/processor/balance_storage_helper.go b/pkg/processor/balance_storage_helper.go index 7374b0f6..c28098b4 100644 --- a/pkg/processor/balance_storage_helper.go +++ b/pkg/processor/balance_storage_helper.go @@ -96,7 +96,7 @@ func (h *BalanceStorageHelper) AccountBalance( // In the case that we are syncing from arbitrary height, // we may need to recover the balance of an account to // perform validations. - amount, block, _, err := utils.CurrencyBalance( + amount, block, err := utils.CurrencyBalance( ctx, h.network, h.fetcher, diff --git a/pkg/processor/reconciler_helper.go b/pkg/processor/reconciler_helper.go index b533ad6d..3fbdbdc5 100644 --- a/pkg/processor/reconciler_helper.go +++ b/pkg/processor/reconciler_helper.go @@ -125,7 +125,7 @@ func (h *ReconcilerHelper) LiveBalance( currency *types.Currency, index int64, ) (*types.Amount, *types.BlockIdentifier, error) { - amt, block, _, err := utils.CurrencyBalance( + amt, block, err := utils.CurrencyBalance( ctx, h.network, h.fetcher, From 792e95919d53d848b9a7328f3b893aaa22265764 Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Mon, 9 Nov 2020 21:04:45 -0800 Subject: [PATCH 193/376] Update version --- cmd/root.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/root.go b/cmd/root.go index 76fbc6e8..58f440a1 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -257,6 +257,6 @@ var versionCmd = &cobra.Command{ Use: "version", Short: "Print rosetta-cli version", Run: func(cmd *cobra.Command, args []string) { - fmt.Println("v0.5.23") + fmt.Println("v0.6.0") }, } From 455dc57deec3b2788e96d4992cc4e72cf07ebede Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Mon, 9 Nov 2020 21:07:08 -0800 Subject: [PATCH 194/376] Update cli commands --- README.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 44055700..175f4599 100644 --- a/README.md +++ b/README.md @@ -60,7 +60,7 @@ Available Commands: utils:asserter-configuration Generate a static configuration file for the Asserter utils:train-zstd Generate a zstd dictionary for enhanced compression performance version Print rosetta-cli version - view:account View an account balance + view:balance View an account balance view:block View a block view:networks View all network statuses @@ -397,22 +397,22 @@ Global Flags: --mem-profile string Save the pprof mem profile in the specified file ``` -#### view:account +#### view:balance ``` While debugging, it is often useful to inspect the state of an account at a certain block. This command allows you to look up any account by providing a JSON representation of a types.AccountIdentifier (and optionally a height to perform the query). -For example, you could run view:account '{"address":"interesting address"}' 1000 +For example, you could run view:balance '{"address":"interesting address"}' 1000 to lookup the balance of an interesting address at block 1000. Allowing the address to specified as JSON allows for querying by SubAccountIdentifier. Usage: - rosetta-cli view:account [flags] + rosetta-cli view:balance [flags] Flags: - -h, --help help for view:account + -h, --help help for view:balance Global Flags: --block-profile string Save the pprof block profile in the specified file From dc57f5e4d9ad6fbb46672a0bdc299548337f81e9 Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Mon, 9 Nov 2020 21:20:25 -0800 Subject: [PATCH 195/376] update rosetta-sdk-go --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index e51f03d0..f862c5a9 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/coinbase/rosetta-cli go 1.13 require ( - github.com/coinbase/rosetta-sdk-go v0.5.11-0.20201110050224-e6be00ef9da1 + github.com/coinbase/rosetta-sdk-go v0.6.0 github.com/fatih/color v1.10.0 github.com/olekukonko/tablewriter v0.0.2-0.20190409134802-7e037d187b0c github.com/spf13/cobra v1.1.1 diff --git a/go.sum b/go.sum index 82da2c2a..a40eac17 100644 --- a/go.sum +++ b/go.sum @@ -76,8 +76,8 @@ github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+ github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cloudflare/cloudflare-go v0.10.2-0.20190916151808-a80f83b9add9/go.mod h1:1MxXX1Ux4x6mqPmjkUgTP1CdXIBXKX7T+Jk9Gxrmx+U= -github.com/coinbase/rosetta-sdk-go v0.5.11-0.20201110050224-e6be00ef9da1 h1:CAceh+0T+53aGGsEf+AKJVxtIaU3pUhEqDN8HrnLXS4= -github.com/coinbase/rosetta-sdk-go v0.5.11-0.20201110050224-e6be00ef9da1/go.mod h1:pqBibzTcpz0mLqb7is8fo2qf93tcHjItJ3u7Yud8RI4= +github.com/coinbase/rosetta-sdk-go v0.6.0 h1:U8/NhkPo7CGJR8Ud82Y0HqtujXSVzGZKscmxOHsmS54= +github.com/coinbase/rosetta-sdk-go v0.6.0/go.mod h1:pqBibzTcpz0mLqb7is8fo2qf93tcHjItJ3u7Yud8RI4= github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= From 2c127486c734a4a667c30ca4e646e3550a85d03f Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Fri, 13 Nov 2020 10:46:42 -0800 Subject: [PATCH 196/376] update linting version --- .circleci/config.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index d0c43111..6db4d75e 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -16,9 +16,9 @@ version: 2.1 executors: default: docker: - - image: circleci/golang:1.13 + - image: circleci/golang:1.15 user: root # go directory is owned by root - working_directory: /go/src/github.com/coinbase/rosetta-sdk-go + working_directory: /go/src/github.com/coinbase/rosetta-cli environment: - GO111MODULE: "on" @@ -50,7 +50,7 @@ jobs: name: default steps: - *fast-checkout - - run: curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(go env GOPATH)/bin v1.30.0 + - run: curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(go env GOPATH)/bin v1.32.2 - run: make lint check-license: executor: From 2a8e8811ad9fb2bdeb364e875c96559175326f4d Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Fri, 13 Nov 2020 10:55:18 -0800 Subject: [PATCH 197/376] update coverage directories --- Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index e716ddc7..84e8fc85 100644 --- a/Makefile +++ b/Makefile @@ -11,8 +11,8 @@ ADDLICENSE_CMD=go run github.com/google/addlicense ADDLICENCE_SCRIPT=${ADDLICENSE_CMD} -c "Coinbase, Inc." -l "apache" -v GOLINES_CMD=go run github.com/segmentio/golines GOVERALLS_CMD=go run github.com/mattn/goveralls -COVERAGE_TEST_DIRECTORIES=./configuration/... ./pkg/constructor/... \ - ./pkg/logger/... ./pkg/scenario/... +COVERAGE_TEST_DIRECTORIES=./configuration/... ./pkg/results/... \ + ./pkg/logger/... TEST_SCRIPT=go test -v ./pkg/... ./configuration/... COVERAGE_TEST_SCRIPT=go test -v ${COVERAGE_TEST_DIRECTORIES} From c908e582f05d3b927f1b76dcb0eda98237dde59b Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Tue, 1 Dec 2020 11:40:28 -0600 Subject: [PATCH 198/376] Add cached count updates --- pkg/processor/reconciler_handler.go | 81 +++++++++++++++++++++++++++-- 1 file changed, 77 insertions(+), 4 deletions(-) diff --git a/pkg/processor/reconciler_handler.go b/pkg/processor/reconciler_handler.go index 2d74ba11..6ce94f44 100644 --- a/pkg/processor/reconciler_handler.go +++ b/pkg/processor/reconciler_handler.go @@ -18,6 +18,8 @@ import ( "context" "fmt" "math/big" + "sync" + "time" "github.com/coinbase/rosetta-cli/pkg/logger" "github.com/coinbase/rosetta-cli/pkg/results" @@ -27,8 +29,22 @@ import ( "github.com/coinbase/rosetta-sdk-go/types" ) +const ( + updateFrequency = 10 * time.Second +) + var _ reconciler.Handler = (*ReconcilerHandler)(nil) +var ( + countKeys = []string{ + storage.FailedReconciliationCounter, + storage.SkippedReconciliationsCounter, + storage.ExemptReconciliationCounter, + storage.ActiveReconciliationCounter, + storage.InactiveReconciliationCounter, + } +) + // ReconcilerHandler implements the Reconciler.Handler interface. type ReconcilerHandler struct { logger *logger.Logger @@ -40,6 +56,9 @@ type ReconcilerHandler struct { InactiveFailureBlock *types.BlockIdentifier ActiveFailureBlock *types.BlockIdentifier + + counterLock sync.Mutex + counts map[string]int64 } // NewReconcilerHandler creates a new ReconcilerHandler. @@ -49,12 +68,55 @@ func NewReconcilerHandler( balanceStorage *storage.BalanceStorage, haltOnReconciliationError bool, ) *ReconcilerHandler { + counts := map[string]int64{} + for _, key := range countKeys { + counts[key] = 0 + } + return &ReconcilerHandler{ logger: logger, counterStorage: counterStorage, balanceStorage: balanceStorage, haltOnReconciliationError: haltOnReconciliationError, + counts: counts, + } +} + +// Updater periodically updates storage with cached counts. +func (h *ReconcilerHandler) Updater(ctx context.Context) error { + tc := time.NewTicker(updateFrequency) + defer tc.Stop() + + for { + select { + case <-tc.C: + if err := h.UpdateCounts(ctx); err != nil { + return err + } + case <-ctx.Done(): + return ctx.Err() + } + } +} + +// UpdateCounts forces cached counts to be written to storage. +func (h *ReconcilerHandler) UpdateCounts(ctx context.Context) error { + for _, key := range countKeys { + h.counterLock.Lock() + count := h.counts[key] + h.counts[key] = 0 + h.counterLock.Unlock() + + if count == 0 { + continue + } + + if _, err := h.counterStorage.Update(ctx, key, big.NewInt(count)); err != nil { + return err + } } + + return nil } // ReconciliationFailed is called each time a reconciliation fails. @@ -69,7 +131,9 @@ func (h *ReconcilerHandler) ReconciliationFailed( liveBalance string, block *types.BlockIdentifier, ) error { - _, _ = h.counterStorage.Update(ctx, storage.FailedReconciliationCounter, big.NewInt(1)) + h.counterLock.Lock() + h.counts[storage.FailedReconciliationCounter]++ + h.counterLock.Unlock() err := h.logger.ReconcileFailureStream( ctx, @@ -85,6 +149,9 @@ func (h *ReconcilerHandler) ReconciliationFailed( } if h.haltOnReconciliationError { + // Update counts before exiting + _ = h.UpdateCounts(ctx) + if reconciliationType == reconciler.InactiveReconciliation { // Populate inactive failure information so we can try to find block with // missing ops. @@ -134,7 +201,9 @@ func (h *ReconcilerHandler) ReconciliationExempt( block *types.BlockIdentifier, exemption *types.BalanceExemption, ) error { - _, _ = h.counterStorage.Update(ctx, storage.ExemptReconciliationCounter, big.NewInt(1)) + h.counterLock.Lock() + h.counts[storage.ExemptReconciliationCounter]++ + h.counterLock.Unlock() // Although the reconciliation was exempt (non-zero difference that was ignored), // we still mark the account as being reconciled because the balance was in the range @@ -154,7 +223,9 @@ func (h *ReconcilerHandler) ReconciliationSkipped( currency *types.Currency, cause string, ) error { - _, _ = h.counterStorage.Update(ctx, storage.SkippedReconciliationsCounter, big.NewInt(1)) + h.counterLock.Lock() + h.counts[storage.SkippedReconciliationsCounter]++ + h.counterLock.Unlock() return nil } @@ -174,7 +245,9 @@ func (h *ReconcilerHandler) ReconciliationSucceeded( counter = storage.InactiveReconciliationCounter } - _, _ = h.counterStorage.Update(ctx, counter, big.NewInt(1)) + h.counterLock.Lock() + h.counts[counter]++ + h.counterLock.Unlock() if err := h.balanceStorage.Reconciled(ctx, account, currency, block); err != nil { return fmt.Errorf("%w: unable to store updated reconciliation", err) From 8d2b9584bc585f921f0be707d20a0c7a51e740ce Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Tue, 1 Dec 2020 11:43:39 -0600 Subject: [PATCH 199/376] Start count updater in check:data --- cmd/check_data.go | 4 ++++ pkg/tester/data.go | 8 ++++++++ 2 files changed, 12 insertions(+) diff --git a/cmd/check_data.go b/cmd/check_data.go index d6c2b7fa..2277ffdd 100644 --- a/cmd/check_data.go +++ b/cmd/check_data.go @@ -142,6 +142,10 @@ func runCheckDataCmd(cmd *cobra.Command, args []string) error { return dataTester.WatchEndConditions(ctx) }) + g.Go(func() error { + return dataTester.StartReconcilerCountUpdater(ctx) + }) + g.Go(func() error { return tester.LogMemoryLoop(ctx) }) diff --git a/pkg/tester/data.go b/pkg/tester/data.go index 068b30ba..8cdb4ee2 100644 --- a/pkg/tester/data.go +++ b/pkg/tester/data.go @@ -372,6 +372,14 @@ func (t *DataTester) StartPruning( return t.syncer.Prune(ctx, t) } +// StartReconcilerCountUpdater attempts to periodically +// write cached reconciler count updates to storage. +func (t *DataTester) StartReconcilerCountUpdater( + ctx context.Context, +) error { + return t.reconcilerHandler.Updater(ctx) +} + // PruneableIndex is the index that is // safe for pruning. func (t *DataTester) PruneableIndex( From 16331d917e44be8b269a8a46581904c034d93fdd Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Tue, 1 Dec 2020 11:45:07 -0600 Subject: [PATCH 200/376] invoke count update before exiting --- pkg/tester/data.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/pkg/tester/data.go b/pkg/tester/data.go index 8cdb4ee2..c851d8f7 100644 --- a/pkg/tester/data.go +++ b/pkg/tester/data.go @@ -720,6 +720,12 @@ func (t *DataTester) WaitForEmptyQueue( return ctx.Err() case <-tc.C: + // We force cached counts to be written before + // determining if we should exit. + if err := t.reconcilerHandler.UpdateCounts(ctx); err != nil { + return err + } + nowComplete, err := t.CompleteReconciliations(ctx) if err != nil { return err From 1077724ae241f937397204d064a1b94a3c737a55 Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Tue, 1 Dec 2020 12:24:16 -0600 Subject: [PATCH 201/376] update rosetta-sdk-go --- go.mod | 2 +- go.sum | 16 ++++++++++++++++ 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index f862c5a9..17bb610b 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/coinbase/rosetta-cli go 1.13 require ( - github.com/coinbase/rosetta-sdk-go v0.6.0 + github.com/coinbase/rosetta-sdk-go v0.6.2-0.20201201181653-fa7c8e42b53e github.com/fatih/color v1.10.0 github.com/olekukonko/tablewriter v0.0.2-0.20190409134802-7e037d187b0c github.com/spf13/cobra v1.1.1 diff --git a/go.sum b/go.sum index a40eac17..38a09137 100644 --- a/go.sum +++ b/go.sum @@ -36,6 +36,7 @@ github.com/StackExchange/wmi v0.0.0-20180116203802-5d049714c4a6 h1:fLjPD/aNc3UIO github.com/StackExchange/wmi v0.0.0-20180116203802-5d049714c4a6/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg= github.com/VictoriaMetrics/fastcache v1.5.7 h1:4y6y0G8PRzszQUYIQHHssv/jgPHAb5qQuuDNdCbyAgw= github.com/VictoriaMetrics/fastcache v1.5.7/go.mod h1:ptDBkNMQI4RtmVo8VS/XwRY6RoTu1dAWCbrk+6WsEM8= +github.com/Zilliqa/gozilliqa-sdk v1.2.1-0.20201201074141-dd0ecada1be6/go.mod h1:eSYp2T6f0apnuW8TzhV3f6Aff2SE8Dwio++U4ha4yEM= github.com/aead/siphash v1.0.1/go.mod h1:Nywa3cDsYNNK3gaciGTWPwHt0wlpNV15vwmswBAUSII= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= @@ -52,11 +53,13 @@ github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+Ce github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= github.com/bketelsen/crypt v0.0.3-0.20200106085610-5cbc8cc4026c/go.mod h1:MKsuJmJgSg28kpZDP6UIiPt0e0Oz0kqKNGyRaWEPv84= github.com/btcsuite/btcd v0.0.0-20171128150713-2e60448ffcc6/go.mod h1:Dmm/EzmjnCiweXmzRIAiUWCInVmPgjkzgv5k4tVyXiQ= +github.com/btcsuite/btcd v0.0.0-20190315201642-aa6e0f35703c/go.mod h1:DrZx5ec/dmnfpw9KyYoQyYo7d0KEvTkk/5M/vbZjAr8= github.com/btcsuite/btcd v0.20.1-beta h1:Ik4hyJqN8Jfyv3S4AGBOmyouMsYE3EdYODkMbQjwPGw= github.com/btcsuite/btcd v0.20.1-beta/go.mod h1:wVuoA8VJLEcwgqHBwHmzLRazpKxTv13Px/pDuV7OomQ= github.com/btcsuite/btcd v0.21.0-beta h1:At9hIZdJW0s9E/fAz28nrz6AmcNlSVucCH796ZteX1M= github.com/btcsuite/btcd v0.21.0-beta/go.mod h1:ZSWyehm27aAuS9bvkATT+Xte3hjHZ+MRgMY/8NJ7K94= github.com/btcsuite/btclog v0.0.0-20170628155309-84c8d2346e9f/go.mod h1:TdznJufoqS23FtqVCzL0ZqgP5MqXbb4fg/WgDys70nA= +github.com/btcsuite/btcutil v0.0.0-20190207003914-4c204d697803/go.mod h1:+5NJ2+qvTyV9exUAL/rxXi3DcLg2Ts+ymUAY5y4NvMg= github.com/btcsuite/btcutil v0.0.0-20190425235716-9e5f4b9a998d/go.mod h1:+5NJ2+qvTyV9exUAL/rxXi3DcLg2Ts+ymUAY5y4NvMg= github.com/btcsuite/btcutil v1.0.2/go.mod h1:j9HUFwoQRsZL3V4n+qG+CUnEGHOarIxfC3Le2Yhbcts= github.com/btcsuite/go-socks v0.0.0-20170105172521-4720035b7bfd/go.mod h1:HHNXQzUsZCxOoE+CPiyCTO6x34Zs86zZUiwtpXoGdtg= @@ -78,6 +81,8 @@ github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDk github.com/cloudflare/cloudflare-go v0.10.2-0.20190916151808-a80f83b9add9/go.mod h1:1MxXX1Ux4x6mqPmjkUgTP1CdXIBXKX7T+Jk9Gxrmx+U= github.com/coinbase/rosetta-sdk-go v0.6.0 h1:U8/NhkPo7CGJR8Ud82Y0HqtujXSVzGZKscmxOHsmS54= github.com/coinbase/rosetta-sdk-go v0.6.0/go.mod h1:pqBibzTcpz0mLqb7is8fo2qf93tcHjItJ3u7Yud8RI4= +github.com/coinbase/rosetta-sdk-go v0.6.2-0.20201201181653-fa7c8e42b53e h1:OWe+e6cS7LvNpLDMabls5yiXEjNwqssYQDM8e0fuACY= +github.com/coinbase/rosetta-sdk-go v0.6.2-0.20201201181653-fa7c8e42b53e/go.mod h1:MvQfsL2KlJ5786OdDviRIJE3agui2YcvS1CaQPDl1Yo= github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= @@ -117,6 +122,7 @@ github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.m github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/ethereum/go-ethereum v1.9.23 h1:SIKhg/z4Q7AbvqcxuPYvMxf36che/Rq/Pp0IdYEkbtw= github.com/ethereum/go-ethereum v1.9.23/go.mod h1:JIfVb6esrqALTExdz9hRYvrP0xBDf6wCncIu1hNwHpM= +github.com/ethereum/go-ethereum v1.9.24/go.mod h1:JIfVb6esrqALTExdz9hRYvrP0xBDf6wCncIu1hNwHpM= github.com/fatih/color v1.3.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/fatih/color v1.10.0 h1:s36xzo75JdqLaaWoiEHk767eHiwo0598uUxyfiPkDsg= @@ -173,11 +179,13 @@ github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXi github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= +github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= github.com/gorilla/websocket v1.4.1-0.20190629185528-ae1634f6a989/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= +github.com/gorilla/websocket v1.4.1/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/graph-gophers/graphql-go v0.0.0-20191115155744-f33e81362277/go.mod h1:9CQHMSxwO4MprSdzoIEobiHpoLtHm77vfxsvsIN5Vuc= github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= @@ -269,6 +277,7 @@ github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3Rllmb github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/naoina/go-stringutil v0.1.0/go.mod h1:XJ2SJL9jCtBh+P9q5btrd/Ylo8XwT/h1USek5+NqSA0= github.com/naoina/toml v0.1.2-0.20170918210437-9fafd6967416/go.mod h1:NBIhNtsFMo3G2szEBne+bO4gS192HuIYRqfvOWb4i1E= +github.com/neilotoole/errgroup v0.1.5/go.mod h1:Q2nLGf+594h0CLBs/Mbg6qOr7GtqDK7C2S41udRnToE= github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= github.com/olekukonko/tablewriter v0.0.1/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo= @@ -315,6 +324,7 @@ github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= +github.com/segmentio/fasthash v1.0.3/go.mod h1:waKX8l2N8yckOgmSsXJi7x1ZfdKZ4x7KRMzBtS3oedY= github.com/shirou/gopsutil v2.20.5+incompatible h1:tYH07UPoQt0OCQdgWWMgYHy3/a9bcxNpBIysykNIP7I= github.com/shirou/gopsutil v2.20.5+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA= github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= @@ -350,12 +360,14 @@ github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+ github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= github.com/syndtr/goleveldb v1.0.1-0.20200815110645-5c35d600f0ca/go.mod h1:u2MKkTVTVJWe5D1rCvame8WqhBd88EuIwODJZ1VHCPM= github.com/tidwall/gjson v1.6.1 h1:LRbvNuNuvAiISWg6gxLEFuCe72UKy5hDqhxW/8183ws= github.com/tidwall/gjson v1.6.1/go.mod h1:BaHyNc5bjzYkPqgLq7mdVzeiRtULKULXLgZFKsxEHI0= +github.com/tidwall/gjson v1.6.3/go.mod h1:BaHyNc5bjzYkPqgLq7mdVzeiRtULKULXLgZFKsxEHI0= github.com/tidwall/match v1.0.1 h1:PnKP62LPNxHKTwvHHZZzdOAOCtsJTjo6dZLCwpKm5xc= github.com/tidwall/match v1.0.1/go.mod h1:LujAq0jyVjBy028G1WhWfIzbpQfMO8bBZ6Tyb0+pL9E= github.com/tidwall/pretty v1.0.2 h1:Z7S3cePv9Jwm1KwS0513MRaoUe3S01WPbLNV40pwWZU= @@ -364,15 +376,18 @@ github.com/tidwall/sjson v1.1.2 h1:NC5okI+tQ8OG/oyzchvwXXxRxCV/FVdhODbPKkQ25jQ= github.com/tidwall/sjson v1.1.2/go.mod h1:SEzaDwxiPzKzNfUEO4HbYF/m4UCSJDsGgNqsS1LvdoY= github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/tyler-smith/go-bip39 v1.0.1-0.20181017060643-dbb3b84ba2ef/go.mod h1:sJ5fKU0s6JVwZjjcUEX2zFOnvq0ASQ2K9Zr6cf67kNs= +github.com/tyler-smith/go-bip39 v1.0.2/go.mod h1:sJ5fKU0s6JVwZjjcUEX2zFOnvq0ASQ2K9Zr6cf67kNs= github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= github.com/vmihailenco/msgpack/v5 v5.0.0-beta.9 h1:iBRIniTnWOo0kqkg3k3XR8Vn6OCkVlIuZNo0UoBrKx4= github.com/vmihailenco/msgpack/v5 v5.0.0-beta.9/go.mod h1:HVxBVPUK/+fZMonk4bi1islLa8V3cfnBug0+4dykPzo= +github.com/vmihailenco/msgpack/v5 v5.1.0/go.mod h1:C5gboKD0TJPqWDTVTtrQNfRbiBwHZGo8UTqP/9/XvLI= github.com/vmihailenco/tagparser v0.1.2 h1:gnjoVuB/kljJ5wICEEOpx98oXMWPLj22G67Vbd1qPqc= github.com/vmihailenco/tagparser v0.1.2/go.mod h1:OeAg3pn3UbLjkWt+rN9oFYB6u/cQgqMEUPoW2WPyhdI= github.com/wsddn/go-ecdh v0.0.0-20161211032359-48726bab9208/go.mod h1:IotVbo4F+mw0EzQ08zFqg7pK3FebNXpaMsRy2RT+Ees= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= +github.com/ybbus/jsonrpc v2.1.2+incompatible/go.mod h1:XJrh1eMSzdIYFbM08flv0wp5G35eRniyeGut1z+LSiE= go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= @@ -384,6 +399,7 @@ golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnf golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20190426145343-a29dc8fdc734/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550 h1:ObdrDkeb4kJdCP557AjRjq69pTHfNouLtWZG7j9rPN8= From 684a66559c8e9e7835f7582377f8facef97b8de5 Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Tue, 1 Dec 2020 12:35:01 -0600 Subject: [PATCH 202/376] Fixing breaking changes --- cmd/utils_train_zstd.go | 2 +- configuration/configuration_test.go | 4 +- configuration/types.go | 4 +- go.sum | 6 ++ mocks/constructor/helper.go | 2 +- pkg/processor/balance_storage_handler.go | 2 +- pkg/processor/balance_storage_helper.go | 2 +- pkg/processor/broadcast_storage_handler.go | 2 +- pkg/processor/broadcast_storage_helper.go | 2 +- pkg/processor/coin_storage_helper.go | 2 +- pkg/processor/coordinator_handler.go | 2 +- pkg/processor/coordinator_helper.go | 2 +- pkg/processor/reconciler_handler.go | 2 +- pkg/processor/reconciler_helper.go | 2 +- pkg/results/construction_results.go | 34 +++++------ pkg/results/data_results.go | 69 +++++++++++----------- pkg/results/data_results_test.go | 38 ++++++------ pkg/tester/construction.go | 2 +- pkg/tester/data.go | 2 +- 19 files changed, 95 insertions(+), 86 deletions(-) diff --git a/cmd/utils_train_zstd.go b/cmd/utils_train_zstd.go index bc715bcd..d3cff75e 100644 --- a/cmd/utils_train_zstd.go +++ b/cmd/utils_train_zstd.go @@ -20,7 +20,7 @@ import ( "path" "strconv" - "github.com/coinbase/rosetta-sdk-go/storage" + "github.com/coinbase/rosetta-sdk-go/storage/database" "github.com/fatih/color" "github.com/spf13/cobra" ) diff --git a/configuration/configuration_test.go b/configuration/configuration_test.go index 7cc2c503..51703403 100644 --- a/configuration/configuration_test.go +++ b/configuration/configuration_test.go @@ -21,7 +21,7 @@ import ( "testing" "github.com/coinbase/rosetta-sdk-go/constructor/job" - "github.com/coinbase/rosetta-sdk-go/storage" + "github.com/coinbase/rosetta-sdk-go/storage/modules" "github.com/coinbase/rosetta-sdk-go/types" "github.com/coinbase/rosetta-sdk-go/utils" "github.com/stretchr/testify/assert" @@ -105,7 +105,7 @@ var ( } invalidPrefundedAccounts = &Configuration{ Construction: &ConstructionConfiguration{ - PrefundedAccounts: []*storage.PrefundedAccount{ + PrefundedAccounts: []*modules.PrefundedAccount{ { PrivateKeyHex: "hello", }, diff --git a/configuration/types.go b/configuration/types.go index 8feb3837..9da2c1fd 100644 --- a/configuration/types.go +++ b/configuration/types.go @@ -16,7 +16,7 @@ package configuration import ( "github.com/coinbase/rosetta-sdk-go/constructor/job" - "github.com/coinbase/rosetta-sdk-go/storage" + "github.com/coinbase/rosetta-sdk-go/storage/modules" "github.com/coinbase/rosetta-sdk-go/types" ) @@ -115,7 +115,7 @@ type ConstructionConfiguration struct { // PrefundedAccounts is an array of prefunded accounts // to use while testing. - PrefundedAccounts []*storage.PrefundedAccount `json:"prefunded_accounts,omitempty"` + PrefundedAccounts []*modules.PrefundedAccount `json:"prefunded_accounts,omitempty"` // Workflows are executed by the rosetta-cli to test // certain construction flows. diff --git a/go.sum b/go.sum index 38a09137..9d05adf0 100644 --- a/go.sum +++ b/go.sum @@ -36,6 +36,7 @@ github.com/StackExchange/wmi v0.0.0-20180116203802-5d049714c4a6 h1:fLjPD/aNc3UIO github.com/StackExchange/wmi v0.0.0-20180116203802-5d049714c4a6/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg= github.com/VictoriaMetrics/fastcache v1.5.7 h1:4y6y0G8PRzszQUYIQHHssv/jgPHAb5qQuuDNdCbyAgw= github.com/VictoriaMetrics/fastcache v1.5.7/go.mod h1:ptDBkNMQI4RtmVo8VS/XwRY6RoTu1dAWCbrk+6WsEM8= +github.com/Zilliqa/gozilliqa-sdk v1.2.1-0.20201201074141-dd0ecada1be6 h1:1d9pzdbkth4D9AX6ndKSl7of3UTV0RYl3z64U2dXMGo= github.com/Zilliqa/gozilliqa-sdk v1.2.1-0.20201201074141-dd0ecada1be6/go.mod h1:eSYp2T6f0apnuW8TzhV3f6Aff2SE8Dwio++U4ha4yEM= github.com/aead/siphash v1.0.1/go.mod h1:Nywa3cDsYNNK3gaciGTWPwHt0wlpNV15vwmswBAUSII= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= @@ -122,6 +123,7 @@ github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.m github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/ethereum/go-ethereum v1.9.23 h1:SIKhg/z4Q7AbvqcxuPYvMxf36che/Rq/Pp0IdYEkbtw= github.com/ethereum/go-ethereum v1.9.23/go.mod h1:JIfVb6esrqALTExdz9hRYvrP0xBDf6wCncIu1hNwHpM= +github.com/ethereum/go-ethereum v1.9.24 h1:6AK+ORt3EMDO+FTjzXy/AQwHMbu52J2nYHIjyQX9azQ= github.com/ethereum/go-ethereum v1.9.24/go.mod h1:JIfVb6esrqALTExdz9hRYvrP0xBDf6wCncIu1hNwHpM= github.com/fatih/color v1.3.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= @@ -277,6 +279,7 @@ github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3Rllmb github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/naoina/go-stringutil v0.1.0/go.mod h1:XJ2SJL9jCtBh+P9q5btrd/Ylo8XwT/h1USek5+NqSA0= github.com/naoina/toml v0.1.2-0.20170918210437-9fafd6967416/go.mod h1:NBIhNtsFMo3G2szEBne+bO4gS192HuIYRqfvOWb4i1E= +github.com/neilotoole/errgroup v0.1.5 h1:DxEGoIfFm5ooGicidR+okiHjoOaGRKFaSxDPVZuuu2I= github.com/neilotoole/errgroup v0.1.5/go.mod h1:Q2nLGf+594h0CLBs/Mbg6qOr7GtqDK7C2S41udRnToE= github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= @@ -324,6 +327,7 @@ github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= +github.com/segmentio/fasthash v1.0.3 h1:EI9+KE1EwvMLBWwjpRDc+fEM+prwxDYbslddQGtrmhM= github.com/segmentio/fasthash v1.0.3/go.mod h1:waKX8l2N8yckOgmSsXJi7x1ZfdKZ4x7KRMzBtS3oedY= github.com/shirou/gopsutil v2.20.5+incompatible h1:tYH07UPoQt0OCQdgWWMgYHy3/a9bcxNpBIysykNIP7I= github.com/shirou/gopsutil v2.20.5+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA= @@ -367,6 +371,7 @@ github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69 github.com/syndtr/goleveldb v1.0.1-0.20200815110645-5c35d600f0ca/go.mod h1:u2MKkTVTVJWe5D1rCvame8WqhBd88EuIwODJZ1VHCPM= github.com/tidwall/gjson v1.6.1 h1:LRbvNuNuvAiISWg6gxLEFuCe72UKy5hDqhxW/8183ws= github.com/tidwall/gjson v1.6.1/go.mod h1:BaHyNc5bjzYkPqgLq7mdVzeiRtULKULXLgZFKsxEHI0= +github.com/tidwall/gjson v1.6.3 h1:aHoiiem0dr7GHkW001T1SMTJ7X5PvyekH5WX0whWGnI= github.com/tidwall/gjson v1.6.3/go.mod h1:BaHyNc5bjzYkPqgLq7mdVzeiRtULKULXLgZFKsxEHI0= github.com/tidwall/match v1.0.1 h1:PnKP62LPNxHKTwvHHZZzdOAOCtsJTjo6dZLCwpKm5xc= github.com/tidwall/match v1.0.1/go.mod h1:LujAq0jyVjBy028G1WhWfIzbpQfMO8bBZ6Tyb0+pL9E= @@ -381,6 +386,7 @@ github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljT github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= github.com/vmihailenco/msgpack/v5 v5.0.0-beta.9 h1:iBRIniTnWOo0kqkg3k3XR8Vn6OCkVlIuZNo0UoBrKx4= github.com/vmihailenco/msgpack/v5 v5.0.0-beta.9/go.mod h1:HVxBVPUK/+fZMonk4bi1islLa8V3cfnBug0+4dykPzo= +github.com/vmihailenco/msgpack/v5 v5.1.0 h1:+od5YbEXxW95SPlW6beocmt8nOtlh83zqat5Ip9Hwdc= github.com/vmihailenco/msgpack/v5 v5.1.0/go.mod h1:C5gboKD0TJPqWDTVTtrQNfRbiBwHZGo8UTqP/9/XvLI= github.com/vmihailenco/tagparser v0.1.2 h1:gnjoVuB/kljJ5wICEEOpx98oXMWPLj22G67Vbd1qPqc= github.com/vmihailenco/tagparser v0.1.2/go.mod h1:OeAg3pn3UbLjkWt+rN9oFYB6u/cQgqMEUPoW2WPyhdI= diff --git a/mocks/constructor/helper.go b/mocks/constructor/helper.go index be7a65cb..91eab4a2 100644 --- a/mocks/constructor/helper.go +++ b/mocks/constructor/helper.go @@ -11,7 +11,7 @@ import ( mock "github.com/stretchr/testify/mock" - storage "github.com/coinbase/rosetta-sdk-go/storage" + storage "github.com/coinbase/rosetta-sdk-go/storage/database" types "github.com/coinbase/rosetta-sdk-go/types" ) diff --git a/pkg/processor/balance_storage_handler.go b/pkg/processor/balance_storage_handler.go index 0e7038fb..10623a02 100644 --- a/pkg/processor/balance_storage_handler.go +++ b/pkg/processor/balance_storage_handler.go @@ -21,7 +21,7 @@ import ( "github.com/coinbase/rosetta-sdk-go/parser" "github.com/coinbase/rosetta-sdk-go/reconciler" - "github.com/coinbase/rosetta-sdk-go/storage" + "github.com/coinbase/rosetta-sdk-go/storage/database" "github.com/coinbase/rosetta-sdk-go/types" ) diff --git a/pkg/processor/balance_storage_helper.go b/pkg/processor/balance_storage_helper.go index c28098b4..52ae1f72 100644 --- a/pkg/processor/balance_storage_helper.go +++ b/pkg/processor/balance_storage_helper.go @@ -21,7 +21,7 @@ import ( "github.com/coinbase/rosetta-sdk-go/asserter" "github.com/coinbase/rosetta-sdk-go/fetcher" "github.com/coinbase/rosetta-sdk-go/parser" - "github.com/coinbase/rosetta-sdk-go/storage" + "github.com/coinbase/rosetta-sdk-go/storage/database" "github.com/coinbase/rosetta-sdk-go/syncer" "github.com/coinbase/rosetta-sdk-go/types" "github.com/coinbase/rosetta-sdk-go/utils" diff --git a/pkg/processor/broadcast_storage_handler.go b/pkg/processor/broadcast_storage_handler.go index 4bfafe78..af778d7c 100644 --- a/pkg/processor/broadcast_storage_handler.go +++ b/pkg/processor/broadcast_storage_handler.go @@ -23,7 +23,7 @@ import ( "github.com/coinbase/rosetta-sdk-go/constructor/coordinator" "github.com/coinbase/rosetta-sdk-go/parser" - "github.com/coinbase/rosetta-sdk-go/storage" + "github.com/coinbase/rosetta-sdk-go/storage/database" "github.com/coinbase/rosetta-sdk-go/types" ) diff --git a/pkg/processor/broadcast_storage_helper.go b/pkg/processor/broadcast_storage_helper.go index f8e4f133..ead6d1a1 100644 --- a/pkg/processor/broadcast_storage_helper.go +++ b/pkg/processor/broadcast_storage_helper.go @@ -19,7 +19,7 @@ import ( "fmt" "github.com/coinbase/rosetta-sdk-go/fetcher" - "github.com/coinbase/rosetta-sdk-go/storage" + "github.com/coinbase/rosetta-sdk-go/storage/database" "github.com/coinbase/rosetta-sdk-go/types" ) diff --git a/pkg/processor/coin_storage_helper.go b/pkg/processor/coin_storage_helper.go index dae4b41a..4e187aee 100644 --- a/pkg/processor/coin_storage_helper.go +++ b/pkg/processor/coin_storage_helper.go @@ -17,7 +17,7 @@ package processor import ( "context" - "github.com/coinbase/rosetta-sdk-go/storage" + "github.com/coinbase/rosetta-sdk-go/storage/database" "github.com/coinbase/rosetta-sdk-go/types" ) diff --git a/pkg/processor/coordinator_handler.go b/pkg/processor/coordinator_handler.go index 2201800d..9200b04d 100644 --- a/pkg/processor/coordinator_handler.go +++ b/pkg/processor/coordinator_handler.go @@ -19,7 +19,7 @@ import ( "math/big" "github.com/coinbase/rosetta-sdk-go/constructor/coordinator" - "github.com/coinbase/rosetta-sdk-go/storage" + "github.com/coinbase/rosetta-sdk-go/storage/database" "github.com/coinbase/rosetta-sdk-go/types" ) diff --git a/pkg/processor/coordinator_helper.go b/pkg/processor/coordinator_helper.go index f21b8a86..847e9ad1 100644 --- a/pkg/processor/coordinator_helper.go +++ b/pkg/processor/coordinator_helper.go @@ -24,7 +24,7 @@ import ( "github.com/coinbase/rosetta-sdk-go/constructor/coordinator" "github.com/coinbase/rosetta-sdk-go/fetcher" "github.com/coinbase/rosetta-sdk-go/keys" - "github.com/coinbase/rosetta-sdk-go/storage" + "github.com/coinbase/rosetta-sdk-go/storage/database" "github.com/coinbase/rosetta-sdk-go/types" ) diff --git a/pkg/processor/reconciler_handler.go b/pkg/processor/reconciler_handler.go index 6ce94f44..4b1748e6 100644 --- a/pkg/processor/reconciler_handler.go +++ b/pkg/processor/reconciler_handler.go @@ -25,7 +25,7 @@ import ( "github.com/coinbase/rosetta-cli/pkg/results" "github.com/coinbase/rosetta-sdk-go/reconciler" - "github.com/coinbase/rosetta-sdk-go/storage" + "github.com/coinbase/rosetta-sdk-go/storage/database" "github.com/coinbase/rosetta-sdk-go/types" ) diff --git a/pkg/processor/reconciler_helper.go b/pkg/processor/reconciler_helper.go index 3fbdbdc5..ebe8157f 100644 --- a/pkg/processor/reconciler_helper.go +++ b/pkg/processor/reconciler_helper.go @@ -21,7 +21,7 @@ import ( "github.com/coinbase/rosetta-sdk-go/fetcher" "github.com/coinbase/rosetta-sdk-go/reconciler" - "github.com/coinbase/rosetta-sdk-go/storage" + "github.com/coinbase/rosetta-sdk-go/storage/database" "github.com/coinbase/rosetta-sdk-go/types" "github.com/coinbase/rosetta-sdk-go/utils" ) diff --git a/pkg/results/construction_results.go b/pkg/results/construction_results.go index 83e44135..22f8b657 100644 --- a/pkg/results/construction_results.go +++ b/pkg/results/construction_results.go @@ -23,7 +23,7 @@ import ( "github.com/coinbase/rosetta-cli/configuration" - "github.com/coinbase/rosetta-sdk-go/storage" + "github.com/coinbase/rosetta-sdk-go/storage/modules" "github.com/coinbase/rosetta-sdk-go/types" "github.com/coinbase/rosetta-sdk-go/utils" "github.com/fatih/color" @@ -73,8 +73,8 @@ func (c *CheckConstructionResults) Output(path string) { func ComputeCheckConstructionResults( cfg *configuration.Configuration, err error, - counterStorage *storage.CounterStorage, - jobStorage *storage.JobStorage, + counterStorage *modules.CounterStorage, + jobStorage *modules.JobStorage, ) *CheckConstructionResults { ctx := context.Background() stats := ComputeCheckConstructionStats(ctx, cfg, counterStorage, jobStorage) @@ -169,38 +169,38 @@ func (c *CheckConstructionStats) Print() { func ComputeCheckConstructionStats( ctx context.Context, config *configuration.Configuration, - counters *storage.CounterStorage, - jobs *storage.JobStorage, + counters *modules.CounterStorage, + jobs *modules.JobStorage, ) *CheckConstructionStats { if counters == nil || jobs == nil { return nil } - transactionsCreated, err := counters.Get(ctx, storage.TransactionsCreatedCounter) + transactionsCreated, err := counters.Get(ctx, modules.TransactionsCreatedCounter) if err != nil { log.Printf("%s cannot get transactions created counter\n", err.Error()) return nil } - transactionsConfirmed, err := counters.Get(ctx, storage.TransactionsConfirmedCounter) + transactionsConfirmed, err := counters.Get(ctx, modules.TransactionsConfirmedCounter) if err != nil { log.Printf("%s cannot get transactions confirmed counter\n", err.Error()) return nil } - staleBroadcasts, err := counters.Get(ctx, storage.StaleBroadcastsCounter) + staleBroadcasts, err := counters.Get(ctx, modules.StaleBroadcastsCounter) if err != nil { log.Printf("%s cannot get stale broadcasts counter\n", err) return nil } - failedBroadcasts, err := counters.Get(ctx, storage.FailedBroadcastsCounter) + failedBroadcasts, err := counters.Get(ctx, modules.FailedBroadcastsCounter) if err != nil { log.Printf("%s cannot get failed broadcasts counter\n", err.Error()) return nil } - addressesCreated, err := counters.Get(ctx, storage.AddressesCreatedCounter) + addressesCreated, err := counters.Get(ctx, modules.AddressesCreatedCounter) if err != nil { log.Printf("%s cannot get addresses created counter\n", err.Error()) return nil @@ -239,8 +239,8 @@ type CheckConstructionProgress struct { // *CheckConstructionProgress. func ComputeCheckConstructionProgress( ctx context.Context, - broadcasts *storage.BroadcastStorage, - jobs *storage.JobStorage, + broadcasts *modules.BroadcastStorage, + jobs *modules.JobStorage, ) *CheckConstructionProgress { inflight, err := broadcasts.GetAllBroadcasts(ctx) if err != nil { @@ -271,9 +271,9 @@ type CheckConstructionStatus struct { func ComputeCheckConstructionStatus( ctx context.Context, config *configuration.Configuration, - counters *storage.CounterStorage, - broadcasts *storage.BroadcastStorage, - jobs *storage.JobStorage, + counters *modules.CounterStorage, + broadcasts *modules.BroadcastStorage, + jobs *modules.JobStorage, ) *CheckConstructionStatus { return &CheckConstructionStatus{ Stats: ComputeCheckConstructionStats(ctx, config, counters, jobs), @@ -295,8 +295,8 @@ func FetchCheckConstructionStatus(url string) (*CheckConstructionStatus, error) // and to a provided output path. func ExitConstruction( config *configuration.Configuration, - counterStorage *storage.CounterStorage, - jobStorage *storage.JobStorage, + counterStorage *modules.CounterStorage, + jobStorage *modules.JobStorage, err error, ) error { results := ComputeCheckConstructionResults( diff --git a/pkg/results/data_results.go b/pkg/results/data_results.go index 950c97cc..5935e524 100644 --- a/pkg/results/data_results.go +++ b/pkg/results/data_results.go @@ -28,7 +28,8 @@ import ( "github.com/coinbase/rosetta-sdk-go/asserter" "github.com/coinbase/rosetta-sdk-go/fetcher" "github.com/coinbase/rosetta-sdk-go/reconciler" - "github.com/coinbase/rosetta-sdk-go/storage" + storageErrs "github.com/coinbase/rosetta-sdk-go/storage/errors" + "github.com/coinbase/rosetta-sdk-go/storage/modules" "github.com/coinbase/rosetta-sdk-go/syncer" "github.com/coinbase/rosetta-sdk-go/types" "github.com/coinbase/rosetta-sdk-go/utils" @@ -175,62 +176,62 @@ func (c *CheckDataStats) Print() { // ComputeCheckDataStats returns a populated CheckDataStats. func ComputeCheckDataStats( ctx context.Context, - counters *storage.CounterStorage, - balances *storage.BalanceStorage, + counters *modules.CounterStorage, + balances *modules.BalanceStorage, ) *CheckDataStats { if counters == nil { return nil } - blocks, err := counters.Get(ctx, storage.BlockCounter) + blocks, err := counters.Get(ctx, modules.BlockCounter) if err != nil { log.Printf("%s: cannot get block counter", err.Error()) return nil } - orphans, err := counters.Get(ctx, storage.OrphanCounter) + orphans, err := counters.Get(ctx, modules.OrphanCounter) if err != nil { log.Printf("%s: cannot get orphan counter", err.Error()) return nil } - txs, err := counters.Get(ctx, storage.TransactionCounter) + txs, err := counters.Get(ctx, modules.TransactionCounter) if err != nil { log.Printf("%s: cannot get transaction counter", err.Error()) return nil } - ops, err := counters.Get(ctx, storage.OperationCounter) + ops, err := counters.Get(ctx, modules.OperationCounter) if err != nil { log.Printf("%s: cannot get operations counter", err.Error()) return nil } - activeReconciliations, err := counters.Get(ctx, storage.ActiveReconciliationCounter) + activeReconciliations, err := counters.Get(ctx, modules.ActiveReconciliationCounter) if err != nil { log.Printf("%s: cannot get active reconciliations counter", err.Error()) return nil } - inactiveReconciliations, err := counters.Get(ctx, storage.InactiveReconciliationCounter) + inactiveReconciliations, err := counters.Get(ctx, modules.InactiveReconciliationCounter) if err != nil { log.Printf("%s: cannot get inactive reconciliations counter", err.Error()) return nil } - exemptReconciliations, err := counters.Get(ctx, storage.ExemptReconciliationCounter) + exemptReconciliations, err := counters.Get(ctx, modules.ExemptReconciliationCounter) if err != nil { log.Printf("%s: cannot get exempt reconciliations counter", err.Error()) return nil } - failedReconciliations, err := counters.Get(ctx, storage.FailedReconciliationCounter) + failedReconciliations, err := counters.Get(ctx, modules.FailedReconciliationCounter) if err != nil { log.Printf("%s: cannot get failed reconciliations counter", err.Error()) return nil } - skippedReconciliations, err := counters.Get(ctx, storage.SkippedReconciliationsCounter) + skippedReconciliations, err := counters.Get(ctx, modules.SkippedReconciliationsCounter) if err != nil { log.Printf("%s: cannot get skipped reconciliations counter", err.Error()) return nil @@ -279,8 +280,8 @@ func ComputeCheckDataProgress( ctx context.Context, fetcher *fetcher.Fetcher, network *types.NetworkIdentifier, - counters *storage.CounterStorage, - blockStorage *storage.BlockStorage, + counters *modules.CounterStorage, + blockStorage *modules.BlockStorage, reconciler *reconciler.Reconciler, ) *CheckDataProgress { networkStatus, fetchErr := fetcher.NetworkStatusRetry(ctx, network, nil) @@ -293,7 +294,7 @@ func ComputeCheckDataProgress( // Get current tip in the case that re-orgs occurred // or a custom start index was provied. headBlock, err := blockStorage.GetHeadBlockIdentifier(ctx) - if errors.Is(err, storage.ErrHeadBlockNotFound) { + if errors.Is(err, storageErrs.ErrHeadBlockNotFound) { return nil } if err != nil { @@ -301,7 +302,7 @@ func ComputeCheckDataProgress( return nil } - blocks, err := counters.Get(ctx, storage.BlockCounter) + blocks, err := counters.Get(ctx, modules.BlockCounter) if err != nil { fmt.Printf("%s: cannot get block counter", err.Error()) return nil @@ -311,7 +312,7 @@ func ComputeCheckDataProgress( return nil } - orphans, err := counters.Get(ctx, storage.OrphanCounter) + orphans, err := counters.Get(ctx, modules.OrphanCounter) if err != nil { fmt.Printf("%s: cannot get orphan counter", err.Error()) return nil @@ -375,9 +376,9 @@ type CheckDataStatus struct { // *CheckDataStatus. func ComputeCheckDataStatus( ctx context.Context, - blocks *storage.BlockStorage, - counters *storage.CounterStorage, - balances *storage.BalanceStorage, + blocks *modules.BlockStorage, + counters *modules.CounterStorage, + balances *modules.BalanceStorage, fetcher *fetcher.Fetcher, network *types.NetworkIdentifier, reconciler *reconciler.Reconciler, @@ -503,9 +504,9 @@ func ResponseAssertionTest(err error) bool { // blocks. func BlockSyncingTest(err error, blocksSynced bool) *bool { syncPass := true - storageFailed, _ := storage.Err(err) + storageFailed, _ := storageErrs.Err(err) if syncer.Err(err) || - (storageFailed && !errors.Is(err, storage.ErrNegativeBalance)) { + (storageFailed && !errors.Is(err, storageErrs.ErrNegativeBalance)) { syncPass = false } @@ -521,7 +522,7 @@ func BlockSyncingTest(err error, blocksSynced bool) *bool { // while syncing. func BalanceTrackingTest(cfg *configuration.Configuration, err error, operationsSeen bool) *bool { balancePass := true - for _, balanceStorageErr := range storage.BalanceStorageErrs { + for _, balanceStorageErr := range storageErrs.BalanceStorageErrs { if errors.Is(err, balanceStorageErr) { balancePass = false break @@ -565,31 +566,31 @@ func ComputeCheckDataTests( // nolint:gocognit ctx context.Context, cfg *configuration.Configuration, err error, - counterStorage *storage.CounterStorage, + counterStorage *modules.CounterStorage, ) *CheckDataTests { operationsSeen := false reconciliationsPerformed := false reconciliationsFailed := false blocksSynced := false if counterStorage != nil { - blocks, err := counterStorage.Get(ctx, storage.BlockCounter) + blocks, err := counterStorage.Get(ctx, modules.BlockCounter) if err == nil && blocks.Int64() > 0 { blocksSynced = true } - ops, err := counterStorage.Get(ctx, storage.OperationCounter) + ops, err := counterStorage.Get(ctx, modules.OperationCounter) if err == nil && ops.Int64() > 0 { operationsSeen = true } - activeReconciliations, err := counterStorage.Get(ctx, storage.ActiveReconciliationCounter) + activeReconciliations, err := counterStorage.Get(ctx, modules.ActiveReconciliationCounter) if err == nil && activeReconciliations.Int64() > 0 { reconciliationsPerformed = true } inactiveReconciliations, err := counterStorage.Get( ctx, - storage.InactiveReconciliationCounter, + modules.InactiveReconciliationCounter, ) if err == nil && inactiveReconciliations.Int64() > 0 { reconciliationsPerformed = true @@ -597,7 +598,7 @@ func ComputeCheckDataTests( // nolint:gocognit exemptReconciliations, err := counterStorage.Get( ctx, - storage.ExemptReconciliationCounter, + modules.ExemptReconciliationCounter, ) if err == nil && exemptReconciliations.Int64() > 0 { reconciliationsPerformed = true @@ -605,7 +606,7 @@ func ComputeCheckDataTests( // nolint:gocognit failedReconciliations, err := counterStorage.Get( ctx, - storage.FailedReconciliationCounter, + modules.FailedReconciliationCounter, ) if err == nil && failedReconciliations.Int64() > 0 { reconciliationsPerformed = true @@ -631,8 +632,8 @@ func ComputeCheckDataTests( // nolint:gocognit func ComputeCheckDataResults( cfg *configuration.Configuration, err error, - counterStorage *storage.CounterStorage, - balanceStorage *storage.BalanceStorage, + counterStorage *modules.CounterStorage, + balanceStorage *modules.BalanceStorage, endCondition configuration.CheckDataEndCondition, endConditionDetail string, ) *CheckDataResults { @@ -677,8 +678,8 @@ func ComputeCheckDataResults( // and to a provided output path. func ExitData( config *configuration.Configuration, - counterStorage *storage.CounterStorage, - balanceStorage *storage.BalanceStorage, + counterStorage *modules.CounterStorage, + balanceStorage *modules.BalanceStorage, err error, endCondition configuration.CheckDataEndCondition, endConditionDetail string, diff --git a/pkg/results/data_results_test.go b/pkg/results/data_results_test.go index 9addec77..ede6bc2e 100644 --- a/pkg/results/data_results_test.go +++ b/pkg/results/data_results_test.go @@ -26,7 +26,9 @@ import ( "github.com/coinbase/rosetta-sdk-go/asserter" "github.com/coinbase/rosetta-sdk-go/fetcher" - "github.com/coinbase/rosetta-sdk-go/storage" + "github.com/coinbase/rosetta-sdk-go/storage/database" + storageErrs "github.com/coinbase/rosetta-sdk-go/storage/errors" + "github.com/coinbase/rosetta-sdk-go/storage/modules" "github.com/coinbase/rosetta-sdk-go/syncer" "github.com/coinbase/rosetta-sdk-go/types" "github.com/coinbase/rosetta-sdk-go/utils" @@ -115,8 +117,8 @@ func TestComputeCheckDataResults(t *testing.T) { err: []error{ syncer.ErrCannotRemoveGenesisBlock, syncer.ErrOutOfOrder, - storage.ErrDuplicateKey, - storage.ErrDuplicateTransactionHash, + storageErrs.ErrDuplicateKey, + storageErrs.ErrDuplicateTransactionHash, }, result: &CheckDataResults{ Tests: &CheckDataTests{ @@ -129,7 +131,7 @@ func TestComputeCheckDataResults(t *testing.T) { "default configuration, counter storage no blocks, balance errors": { cfg: configuration.DefaultConfiguration(), provideCounterStorage: true, - err: []error{storage.ErrNegativeBalance}, + err: []error{storageErrs.ErrNegativeBalance}, result: &CheckDataResults{ Tests: &CheckDataTests{ RequestResponse: true, @@ -143,7 +145,7 @@ func TestComputeCheckDataResults(t *testing.T) { cfg: configuration.DefaultConfiguration(), provideCounterStorage: true, blockCount: 100, - err: []error{storage.ErrNegativeBalance}, + err: []error{storageErrs.ErrNegativeBalance}, result: &CheckDataResults{ Tests: &CheckDataTests{ RequestResponse: true, @@ -279,7 +281,7 @@ func TestComputeCheckDataResults(t *testing.T) { }, "default configuration, no storage, balance errors": { cfg: configuration.DefaultConfiguration(), - err: []error{storage.ErrNegativeBalance}, + err: []error{storageErrs.ErrNegativeBalance}, result: &CheckDataResults{ Tests: &CheckDataTests{ RequestResponse: true, @@ -358,63 +360,63 @@ func TestComputeCheckDataResults(t *testing.T) { assert.NoError(t, err) ctx := context.Background() - localStore, err := storage.NewBadgerStorage( + localStore, err := database.NewBadgerDatabase( ctx, dir, - storage.WithIndexCacheSize(storage.TinyIndexCacheSize), + database.WithIndexCacheSize(database.TinyIndexCacheSize), ) assert.NoError(t, err) logPath := path.Join(dir, "results.json") - var counterStorage *storage.CounterStorage + var counterStorage *modules.CounterStorage if test.provideCounterStorage { - counterStorage = storage.NewCounterStorage(localStore) + counterStorage = modules.NewCounterStorage(localStore) _, err = counterStorage.Update( ctx, - storage.BlockCounter, + modules.BlockCounter, big.NewInt(test.blockCount), ) assert.NoError(t, err) _, err = counterStorage.Update( ctx, - storage.OperationCounter, + modules.OperationCounter, big.NewInt(test.operationCount), ) assert.NoError(t, err) _, err = counterStorage.Update( ctx, - storage.ActiveReconciliationCounter, + modules.ActiveReconciliationCounter, big.NewInt(test.activeReconciliations), ) assert.NoError(t, err) _, err = counterStorage.Update( ctx, - storage.InactiveReconciliationCounter, + modules.InactiveReconciliationCounter, big.NewInt(test.inactiveReconciliations), ) assert.NoError(t, err) _, err = counterStorage.Update( ctx, - storage.FailedReconciliationCounter, + modules.FailedReconciliationCounter, big.NewInt(test.reconciliationFailures), ) assert.NoError(t, err) } - var balanceStorage *storage.BalanceStorage + var balanceStorage *modules.BalanceStorage if test.provideBalanceStorage { - balanceStorage = storage.NewBalanceStorage(localStore) + balanceStorage = modules.NewBalanceStorage(localStore) j := 0 currency := &types.Currency{Symbol: "BLAH"} block := &types.BlockIdentifier{Hash: "0", Index: 0} for i := 0; i < test.totalAccounts; i++ { - dbTransaction := localStore.NewDatabaseTransaction(ctx, true) + dbTransaction := localStore.Transaction(ctx) acct := &types.AccountIdentifier{ Address: fmt.Sprintf("account %d", i), } diff --git a/pkg/tester/construction.go b/pkg/tester/construction.go index 194198ae..edf0b590 100644 --- a/pkg/tester/construction.go +++ b/pkg/tester/construction.go @@ -32,7 +32,7 @@ import ( "github.com/coinbase/rosetta-sdk-go/fetcher" "github.com/coinbase/rosetta-sdk-go/parser" "github.com/coinbase/rosetta-sdk-go/statefulsyncer" - "github.com/coinbase/rosetta-sdk-go/storage" + "github.com/coinbase/rosetta-sdk-go/storage/database" "github.com/coinbase/rosetta-sdk-go/syncer" "github.com/coinbase/rosetta-sdk-go/types" "github.com/coinbase/rosetta-sdk-go/utils" diff --git a/pkg/tester/data.go b/pkg/tester/data.go index c851d8f7..048fe1b5 100644 --- a/pkg/tester/data.go +++ b/pkg/tester/data.go @@ -33,7 +33,7 @@ import ( "github.com/coinbase/rosetta-sdk-go/parser" "github.com/coinbase/rosetta-sdk-go/reconciler" "github.com/coinbase/rosetta-sdk-go/statefulsyncer" - "github.com/coinbase/rosetta-sdk-go/storage" + "github.com/coinbase/rosetta-sdk-go/storage/database" "github.com/coinbase/rosetta-sdk-go/syncer" "github.com/coinbase/rosetta-sdk-go/types" "github.com/coinbase/rosetta-sdk-go/utils" From 55e58332260a85a4d287284d9e83585b07fe0a44 Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Tue, 1 Dec 2020 12:45:33 -0600 Subject: [PATCH 203/376] update storage breaking changes --- go.sum | 1 + pkg/processor/balance_storage_handler.go | 4 +- pkg/processor/balance_storage_helper.go | 4 +- pkg/processor/broadcast_storage_handler.go | 19 ++++---- pkg/processor/broadcast_storage_helper.go | 9 ++-- pkg/processor/coin_storage_helper.go | 9 ++-- pkg/processor/coordinator_handler.go | 8 ++-- pkg/processor/coordinator_helper.go | 55 +++++++++++----------- pkg/processor/reconciler_handler.go | 34 ++++++------- pkg/processor/reconciler_helper.go | 25 +++++----- 10 files changed, 87 insertions(+), 81 deletions(-) diff --git a/go.sum b/go.sum index 9d05adf0..22792f07 100644 --- a/go.sum +++ b/go.sum @@ -82,6 +82,7 @@ github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDk github.com/cloudflare/cloudflare-go v0.10.2-0.20190916151808-a80f83b9add9/go.mod h1:1MxXX1Ux4x6mqPmjkUgTP1CdXIBXKX7T+Jk9Gxrmx+U= github.com/coinbase/rosetta-sdk-go v0.6.0 h1:U8/NhkPo7CGJR8Ud82Y0HqtujXSVzGZKscmxOHsmS54= github.com/coinbase/rosetta-sdk-go v0.6.0/go.mod h1:pqBibzTcpz0mLqb7is8fo2qf93tcHjItJ3u7Yud8RI4= +github.com/coinbase/rosetta-sdk-go v0.6.1 h1:aOb5qstlX0uqP9HRC7wCY+YAZDzZbS2C/i3Qy/lR3xM= github.com/coinbase/rosetta-sdk-go v0.6.2-0.20201201181653-fa7c8e42b53e h1:OWe+e6cS7LvNpLDMabls5yiXEjNwqssYQDM8e0fuACY= github.com/coinbase/rosetta-sdk-go v0.6.2-0.20201201181653-fa7c8e42b53e/go.mod h1:MvQfsL2KlJ5786OdDviRIJE3agui2YcvS1CaQPDl1Yo= github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= diff --git a/pkg/processor/balance_storage_handler.go b/pkg/processor/balance_storage_handler.go index 10623a02..4c054db0 100644 --- a/pkg/processor/balance_storage_handler.go +++ b/pkg/processor/balance_storage_handler.go @@ -21,11 +21,11 @@ import ( "github.com/coinbase/rosetta-sdk-go/parser" "github.com/coinbase/rosetta-sdk-go/reconciler" - "github.com/coinbase/rosetta-sdk-go/storage/database" + "github.com/coinbase/rosetta-sdk-go/storage/modules" "github.com/coinbase/rosetta-sdk-go/types" ) -var _ storage.BalanceStorageHandler = (*BalanceStorageHandler)(nil) +var _ modules.BalanceStorageHandler = (*BalanceStorageHandler)(nil) // BalanceStorageHandler is invoked whenever a block is added // or removed from block storage so that balance changes diff --git a/pkg/processor/balance_storage_helper.go b/pkg/processor/balance_storage_helper.go index 52ae1f72..9648449d 100644 --- a/pkg/processor/balance_storage_helper.go +++ b/pkg/processor/balance_storage_helper.go @@ -21,13 +21,13 @@ import ( "github.com/coinbase/rosetta-sdk-go/asserter" "github.com/coinbase/rosetta-sdk-go/fetcher" "github.com/coinbase/rosetta-sdk-go/parser" - "github.com/coinbase/rosetta-sdk-go/storage/database" + "github.com/coinbase/rosetta-sdk-go/storage/modules" "github.com/coinbase/rosetta-sdk-go/syncer" "github.com/coinbase/rosetta-sdk-go/types" "github.com/coinbase/rosetta-sdk-go/utils" ) -var _ storage.BalanceStorageHelper = (*BalanceStorageHelper)(nil) +var _ modules.BalanceStorageHelper = (*BalanceStorageHelper)(nil) // BalanceStorageHelper implements the storage.Helper // interface. diff --git a/pkg/processor/broadcast_storage_handler.go b/pkg/processor/broadcast_storage_handler.go index af778d7c..57b881c3 100644 --- a/pkg/processor/broadcast_storage_handler.go +++ b/pkg/processor/broadcast_storage_handler.go @@ -24,17 +24,18 @@ import ( "github.com/coinbase/rosetta-sdk-go/constructor/coordinator" "github.com/coinbase/rosetta-sdk-go/parser" "github.com/coinbase/rosetta-sdk-go/storage/database" + "github.com/coinbase/rosetta-sdk-go/storage/modules" "github.com/coinbase/rosetta-sdk-go/types" ) -var _ storage.BroadcastStorageHandler = (*BroadcastStorageHandler)(nil) +var _ modules.BroadcastStorageHandler = (*BroadcastStorageHandler)(nil) // BroadcastStorageHandler is invoked whenever a block is added // or removed from block storage so that balance changes // can be sent to other functions (ex: reconciler). type BroadcastStorageHandler struct { config *configuration.Configuration - counterStorage *storage.CounterStorage + counterStorage *modules.CounterStorage coordinator *coordinator.Coordinator parser *parser.Parser } @@ -42,7 +43,7 @@ type BroadcastStorageHandler struct { // NewBroadcastStorageHandler returns a new *BroadcastStorageHandler. func NewBroadcastStorageHandler( config *configuration.Configuration, - counterStorage *storage.CounterStorage, + counterStorage *modules.CounterStorage, coordinator *coordinator.Coordinator, parser *parser.Parser, ) *BroadcastStorageHandler { @@ -58,7 +59,7 @@ func NewBroadcastStorageHandler( // last time at a block height < current block height - confirmationDepth. func (h *BroadcastStorageHandler) TransactionConfirmed( ctx context.Context, - dbTx storage.DatabaseTransaction, + dbTx database.Transaction, identifier string, blockIdentifier *types.BlockIdentifier, transaction *types.Transaction, @@ -71,7 +72,7 @@ func (h *BroadcastStorageHandler) TransactionConfirmed( _, _ = h.counterStorage.UpdateTransactional( ctx, dbTx, - storage.TransactionsConfirmedCounter, + modules.TransactionsConfirmedCounter, big.NewInt(1), ) @@ -92,14 +93,14 @@ func (h *BroadcastStorageHandler) TransactionConfirmed( // current block height - last broadcast > staleDepth. func (h *BroadcastStorageHandler) TransactionStale( ctx context.Context, - dbTx storage.DatabaseTransaction, + dbTx database.Transaction, identifier string, transactionIdentifier *types.TransactionIdentifier, ) error { _, _ = h.counterStorage.UpdateTransactional( ctx, dbTx, - storage.StaleBroadcastsCounter, + modules.StaleBroadcastsCounter, big.NewInt(1), ) @@ -110,7 +111,7 @@ func (h *BroadcastStorageHandler) TransactionStale( // put it over the provided broadcast limit. func (h *BroadcastStorageHandler) BroadcastFailed( ctx context.Context, - dbTx storage.DatabaseTransaction, + dbTx database.Transaction, identifier string, transactionIdentifier *types.TransactionIdentifier, intent []*types.Operation, @@ -118,7 +119,7 @@ func (h *BroadcastStorageHandler) BroadcastFailed( _, _ = h.counterStorage.UpdateTransactional( ctx, dbTx, - storage.FailedBroadcastsCounter, + modules.FailedBroadcastsCounter, big.NewInt(1), ) diff --git a/pkg/processor/broadcast_storage_helper.go b/pkg/processor/broadcast_storage_helper.go index ead6d1a1..91eecb2f 100644 --- a/pkg/processor/broadcast_storage_helper.go +++ b/pkg/processor/broadcast_storage_helper.go @@ -20,21 +20,22 @@ import ( "github.com/coinbase/rosetta-sdk-go/fetcher" "github.com/coinbase/rosetta-sdk-go/storage/database" + "github.com/coinbase/rosetta-sdk-go/storage/modules" "github.com/coinbase/rosetta-sdk-go/types" ) -var _ storage.BroadcastStorageHelper = (*BroadcastStorageHelper)(nil) +var _ modules.BroadcastStorageHelper = (*BroadcastStorageHelper)(nil) // BroadcastStorageHelper implements the storage.Helper // interface. type BroadcastStorageHelper struct { - blockStorage *storage.BlockStorage + blockStorage *modules.BlockStorage fetcher *fetcher.Fetcher } // NewBroadcastStorageHelper returns a new BroadcastStorageHelper. func NewBroadcastStorageHelper( - blockStorage *storage.BlockStorage, + blockStorage *modules.BlockStorage, fetcher *fetcher.Fetcher, ) *BroadcastStorageHelper { return &BroadcastStorageHelper{ @@ -75,7 +76,7 @@ func (h *BroadcastStorageHelper) CurrentBlockIdentifier( func (h *BroadcastStorageHelper) FindTransaction( ctx context.Context, transactionIdentifier *types.TransactionIdentifier, - txn storage.DatabaseTransaction, + txn database.Transaction, ) (*types.BlockIdentifier, *types.Transaction, error) { newestBlock, transaction, err := h.blockStorage.FindTransaction(ctx, transactionIdentifier, txn) if err != nil { diff --git a/pkg/processor/coin_storage_helper.go b/pkg/processor/coin_storage_helper.go index 4e187aee..afad8c5f 100644 --- a/pkg/processor/coin_storage_helper.go +++ b/pkg/processor/coin_storage_helper.go @@ -18,19 +18,20 @@ import ( "context" "github.com/coinbase/rosetta-sdk-go/storage/database" + "github.com/coinbase/rosetta-sdk-go/storage/modules" "github.com/coinbase/rosetta-sdk-go/types" ) -var _ storage.CoinStorageHelper = (*CoinStorageHelper)(nil) +var _ modules.CoinStorageHelper = (*CoinStorageHelper)(nil) // CoinStorageHelper implements the storage.CoinStorageHelper // interface. type CoinStorageHelper struct { - blockStorage *storage.BlockStorage + blockStorage *modules.BlockStorage } // NewCoinStorageHelper returns a new *CoinStorageHelper. -func NewCoinStorageHelper(blockStorage *storage.BlockStorage) *CoinStorageHelper { +func NewCoinStorageHelper(blockStorage *modules.BlockStorage) *CoinStorageHelper { return &CoinStorageHelper{blockStorage: blockStorage} } @@ -38,7 +39,7 @@ func NewCoinStorageHelper(blockStorage *storage.BlockStorage) *CoinStorageHelper // the context of a storage.DatabaseTransaction. func (c *CoinStorageHelper) CurrentBlockIdentifier( ctx context.Context, - transaction storage.DatabaseTransaction, + transaction database.Transaction, ) (*types.BlockIdentifier, error) { return c.blockStorage.GetHeadBlockIdentifierTransactional(ctx, transaction) } diff --git a/pkg/processor/coordinator_handler.go b/pkg/processor/coordinator_handler.go index 9200b04d..e78ae765 100644 --- a/pkg/processor/coordinator_handler.go +++ b/pkg/processor/coordinator_handler.go @@ -19,7 +19,7 @@ import ( "math/big" "github.com/coinbase/rosetta-sdk-go/constructor/coordinator" - "github.com/coinbase/rosetta-sdk-go/storage/database" + "github.com/coinbase/rosetta-sdk-go/storage/modules" "github.com/coinbase/rosetta-sdk-go/types" ) @@ -28,13 +28,13 @@ var _ coordinator.Handler = (*CoordinatorHandler)(nil) // CoordinatorHandler is invoked by the Coordinator // when addresses are created or transactions are created. type CoordinatorHandler struct { - counterStorage *storage.CounterStorage + counterStorage *modules.CounterStorage } // NewCoordinatorHandler returns a new // *CoordinatorHandler. func NewCoordinatorHandler( - counterStorage *storage.CounterStorage, + counterStorage *modules.CounterStorage, ) *CoordinatorHandler { return &CoordinatorHandler{ counterStorage: counterStorage, @@ -50,7 +50,7 @@ func (h *CoordinatorHandler) TransactionCreated( ) error { _, _ = h.counterStorage.Update( ctx, - storage.TransactionsCreatedCounter, + modules.TransactionsCreatedCounter, big.NewInt(1), ) diff --git a/pkg/processor/coordinator_helper.go b/pkg/processor/coordinator_helper.go index 847e9ad1..c017e8c9 100644 --- a/pkg/processor/coordinator_helper.go +++ b/pkg/processor/coordinator_helper.go @@ -25,6 +25,7 @@ import ( "github.com/coinbase/rosetta-sdk-go/fetcher" "github.com/coinbase/rosetta-sdk-go/keys" "github.com/coinbase/rosetta-sdk-go/storage/database" + "github.com/coinbase/rosetta-sdk-go/storage/modules" "github.com/coinbase/rosetta-sdk-go/types" ) @@ -62,13 +63,13 @@ type CoordinatorHelper struct { offlineFetcher *fetcher.Fetcher onlineFetcher *fetcher.Fetcher - database storage.Database - blockStorage *storage.BlockStorage - keyStorage *storage.KeyStorage - balanceStorage *storage.BalanceStorage - coinStorage *storage.CoinStorage - broadcastStorage *storage.BroadcastStorage - counterStorage *storage.CounterStorage + database database.Database + blockStorage *modules.BlockStorage + keyStorage *modules.KeyStorage + balanceStorage *modules.BalanceStorage + coinStorage *modules.CoinStorage + broadcastStorage *modules.BroadcastStorage + counterStorage *modules.CounterStorage balanceStorageHelper *BalanceStorageHelper @@ -81,14 +82,14 @@ type CoordinatorHelper struct { func NewCoordinatorHelper( offlineFetcher *fetcher.Fetcher, onlineFetcher *fetcher.Fetcher, - database storage.Database, - blockStorage *storage.BlockStorage, - keyStorage *storage.KeyStorage, - balanceStorage *storage.BalanceStorage, - coinStorage *storage.CoinStorage, - broadcastStorage *storage.BroadcastStorage, + database database.Database, + blockStorage *modules.BlockStorage, + keyStorage *modules.KeyStorage, + balanceStorage *modules.BalanceStorage, + coinStorage *modules.CoinStorage, + broadcastStorage *modules.BroadcastStorage, balanceStorageHelper *BalanceStorageHelper, - counterStorage *storage.CounterStorage, + counterStorage *modules.CounterStorage, quiet bool, ) *CoordinatorHelper { return &CoordinatorHelper{ @@ -106,9 +107,9 @@ func NewCoordinatorHelper( } } -// DatabaseTransaction returns a new write-ready storage.DatabaseTransaction. -func (c *CoordinatorHelper) DatabaseTransaction(ctx context.Context) storage.DatabaseTransaction { - return c.database.NewDatabaseTransaction(ctx, true) +// DatabaseTransaction returns a new write-ready database.Transaction. +func (c *CoordinatorHelper) DatabaseTransaction(ctx context.Context) database.Transaction { + return c.database.Transaction(ctx) } type arg struct { @@ -359,7 +360,7 @@ func (c *CoordinatorHelper) Sign( // associated with an address. func (c *CoordinatorHelper) GetKey( ctx context.Context, - dbTx storage.DatabaseTransaction, + dbTx database.Transaction, account *types.AccountIdentifier, ) (*keys.KeyPair, error) { return c.keyStorage.GetTransactional(ctx, dbTx, account) @@ -369,7 +370,7 @@ func (c *CoordinatorHelper) GetKey( // in KeyStorage. func (c *CoordinatorHelper) StoreKey( ctx context.Context, - dbTx storage.DatabaseTransaction, + dbTx database.Transaction, account *types.AccountIdentifier, keyPair *keys.KeyPair, ) error { @@ -379,7 +380,7 @@ func (c *CoordinatorHelper) StoreKey( _, _ = c.counterStorage.UpdateTransactional( ctx, dbTx, - storage.AddressesCreatedCounter, + modules.AddressesCreatedCounter, big.NewInt(1), ) return c.keyStorage.StoreTransactional(ctx, account, keyPair, dbTx) @@ -391,7 +392,7 @@ func (c *CoordinatorHelper) StoreKey( // 0 will be returned. func (c *CoordinatorHelper) Balance( ctx context.Context, - dbTx storage.DatabaseTransaction, + dbTx database.Transaction, accountIdentifier *types.AccountIdentifier, currency *types.Currency, ) (*types.Amount, error) { @@ -416,7 +417,7 @@ func (c *CoordinatorHelper) Balance( // an account. func (c *CoordinatorHelper) Coins( ctx context.Context, - dbTx storage.DatabaseTransaction, + dbTx database.Transaction, accountIdentifier *types.AccountIdentifier, currency *types.Currency, ) ([]*types.Coin, error) { @@ -445,25 +446,25 @@ func (c *CoordinatorHelper) Coins( // funds. func (c *CoordinatorHelper) LockedAccounts( ctx context.Context, - dbTx storage.DatabaseTransaction, + dbTx database.Transaction, ) ([]*types.AccountIdentifier, error) { return c.broadcastStorage.LockedAccounts(ctx, dbTx) } // AllBroadcasts returns a slice of all in-progress broadcasts in BroadcastStorage. -func (c *CoordinatorHelper) AllBroadcasts(ctx context.Context) ([]*storage.Broadcast, error) { +func (c *CoordinatorHelper) AllBroadcasts(ctx context.Context) ([]*modules.Broadcast, error) { return c.broadcastStorage.GetAllBroadcasts(ctx) } // ClearBroadcasts deletes all pending broadcasts. -func (c *CoordinatorHelper) ClearBroadcasts(ctx context.Context) ([]*storage.Broadcast, error) { +func (c *CoordinatorHelper) ClearBroadcasts(ctx context.Context) ([]*modules.Broadcast, error) { return c.broadcastStorage.ClearBroadcasts(ctx) } // Broadcast enqueues a particular intent for broadcast. func (c *CoordinatorHelper) Broadcast( ctx context.Context, - dbTx storage.DatabaseTransaction, + dbTx database.Transaction, identifier string, network *types.NetworkIdentifier, intent []*types.Operation, @@ -499,7 +500,7 @@ func (c *CoordinatorHelper) BroadcastAll( // AllAccounts returns a slice of all known accounts. func (c *CoordinatorHelper) AllAccounts( ctx context.Context, - dbTx storage.DatabaseTransaction, + dbTx database.Transaction, ) ([]*types.AccountIdentifier, error) { return c.keyStorage.GetAllAccountsTransactional(ctx, dbTx) } diff --git a/pkg/processor/reconciler_handler.go b/pkg/processor/reconciler_handler.go index 4b1748e6..13521500 100644 --- a/pkg/processor/reconciler_handler.go +++ b/pkg/processor/reconciler_handler.go @@ -25,7 +25,7 @@ import ( "github.com/coinbase/rosetta-cli/pkg/results" "github.com/coinbase/rosetta-sdk-go/reconciler" - "github.com/coinbase/rosetta-sdk-go/storage/database" + "github.com/coinbase/rosetta-sdk-go/storage/modules" "github.com/coinbase/rosetta-sdk-go/types" ) @@ -37,19 +37,19 @@ var _ reconciler.Handler = (*ReconcilerHandler)(nil) var ( countKeys = []string{ - storage.FailedReconciliationCounter, - storage.SkippedReconciliationsCounter, - storage.ExemptReconciliationCounter, - storage.ActiveReconciliationCounter, - storage.InactiveReconciliationCounter, + modules.FailedReconciliationCounter, + modules.SkippedReconciliationsCounter, + modules.ExemptReconciliationCounter, + modules.ActiveReconciliationCounter, + modules.InactiveReconciliationCounter, } ) // ReconcilerHandler implements the Reconciler.Handler interface. type ReconcilerHandler struct { logger *logger.Logger - counterStorage *storage.CounterStorage - balanceStorage *storage.BalanceStorage + counterStorage *modules.CounterStorage + balanceStorage *modules.BalanceStorage haltOnReconciliationError bool InactiveFailure *types.AccountCurrency @@ -64,8 +64,8 @@ type ReconcilerHandler struct { // NewReconcilerHandler creates a new ReconcilerHandler. func NewReconcilerHandler( logger *logger.Logger, - counterStorage *storage.CounterStorage, - balanceStorage *storage.BalanceStorage, + counterStorage *modules.CounterStorage, + balanceStorage *modules.BalanceStorage, haltOnReconciliationError bool, ) *ReconcilerHandler { counts := map[string]int64{} @@ -82,7 +82,7 @@ func NewReconcilerHandler( } } -// Updater periodically updates storage with cached counts. +// Updater periodically updates modules.with cached counts. func (h *ReconcilerHandler) Updater(ctx context.Context) error { tc := time.NewTicker(updateFrequency) defer tc.Stop() @@ -99,7 +99,7 @@ func (h *ReconcilerHandler) Updater(ctx context.Context) error { } } -// UpdateCounts forces cached counts to be written to storage. +// UpdateCounts forces cached counts to be written to modules. func (h *ReconcilerHandler) UpdateCounts(ctx context.Context) error { for _, key := range countKeys { h.counterLock.Lock() @@ -132,7 +132,7 @@ func (h *ReconcilerHandler) ReconciliationFailed( block *types.BlockIdentifier, ) error { h.counterLock.Lock() - h.counts[storage.FailedReconciliationCounter]++ + h.counts[modules.FailedReconciliationCounter]++ h.counterLock.Unlock() err := h.logger.ReconcileFailureStream( @@ -202,7 +202,7 @@ func (h *ReconcilerHandler) ReconciliationExempt( exemption *types.BalanceExemption, ) error { h.counterLock.Lock() - h.counts[storage.ExemptReconciliationCounter]++ + h.counts[modules.ExemptReconciliationCounter]++ h.counterLock.Unlock() // Although the reconciliation was exempt (non-zero difference that was ignored), @@ -224,7 +224,7 @@ func (h *ReconcilerHandler) ReconciliationSkipped( cause string, ) error { h.counterLock.Lock() - h.counts[storage.SkippedReconciliationsCounter]++ + h.counts[modules.SkippedReconciliationsCounter]++ h.counterLock.Unlock() return nil @@ -240,9 +240,9 @@ func (h *ReconcilerHandler) ReconciliationSucceeded( block *types.BlockIdentifier, ) error { // Update counters - counter := storage.ActiveReconciliationCounter + counter := modules.ActiveReconciliationCounter if reconciliationType == reconciler.InactiveReconciliation { - counter = storage.InactiveReconciliationCounter + counter = modules.InactiveReconciliationCounter } h.counterLock.Lock() diff --git a/pkg/processor/reconciler_helper.go b/pkg/processor/reconciler_helper.go index ebe8157f..5254d4b3 100644 --- a/pkg/processor/reconciler_helper.go +++ b/pkg/processor/reconciler_helper.go @@ -22,6 +22,7 @@ import ( "github.com/coinbase/rosetta-sdk-go/fetcher" "github.com/coinbase/rosetta-sdk-go/reconciler" "github.com/coinbase/rosetta-sdk-go/storage/database" + "github.com/coinbase/rosetta-sdk-go/storage/modules" "github.com/coinbase/rosetta-sdk-go/types" "github.com/coinbase/rosetta-sdk-go/utils" ) @@ -36,9 +37,9 @@ type ReconcilerHelper struct { network *types.NetworkIdentifier fetcher *fetcher.Fetcher - database storage.Database - blockStorage *storage.BlockStorage - balanceStorage *storage.BalanceStorage + database database.Database + blockStorage *modules.BlockStorage + balanceStorage *modules.BalanceStorage } // NewReconcilerHelper returns a new ReconcilerHelper. @@ -46,9 +47,9 @@ func NewReconcilerHelper( config *configuration.Configuration, network *types.NetworkIdentifier, fetcher *fetcher.Fetcher, - database storage.Database, - blockStorage *storage.BlockStorage, - balanceStorage *storage.BalanceStorage, + database database.Database, + blockStorage *modules.BlockStorage, + balanceStorage *modules.BalanceStorage, ) *ReconcilerHelper { return &ReconcilerHelper{ config: config, @@ -60,11 +61,11 @@ func NewReconcilerHelper( } } -// DatabaseTransaction returns a new read-only storage.DatabaseTransaction. +// DatabaseTransaction returns a new read-only database.Transaction. func (h *ReconcilerHelper) DatabaseTransaction( ctx context.Context, -) storage.DatabaseTransaction { - return h.database.NewDatabaseTransaction(ctx, false) +) database.Transaction { + return h.database.ReadTransaction(ctx) } // CanonicalBlock returns a boolean indicating if a block @@ -73,7 +74,7 @@ func (h *ReconcilerHelper) DatabaseTransaction( // does not exist, reconciliation will be skipped. func (h *ReconcilerHelper) CanonicalBlock( ctx context.Context, - dbTx storage.DatabaseTransaction, + dbTx database.Transaction, block *types.BlockIdentifier, ) (bool, error) { return h.blockStorage.CanonicalBlockTransactional(ctx, block, dbTx) @@ -100,7 +101,7 @@ func (h *ReconcilerHelper) IndexAtTip( // inactive reconciliation. func (h *ReconcilerHelper) CurrentBlock( ctx context.Context, - dbTx storage.DatabaseTransaction, + dbTx database.Transaction, ) (*types.BlockIdentifier, error) { return h.blockStorage.GetHeadBlockIdentifierTransactional(ctx, dbTx) } @@ -110,7 +111,7 @@ func (h *ReconcilerHelper) CurrentBlock( // package to allow for separation from a default storage backend. func (h *ReconcilerHelper) ComputedBalance( ctx context.Context, - dbTx storage.DatabaseTransaction, + dbTx database.Transaction, account *types.AccountIdentifier, currency *types.Currency, index int64, From 7c993bd77740d47a0d52bec4dd97b62df6e6b230 Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Tue, 1 Dec 2020 12:52:01 -0600 Subject: [PATCH 204/376] fix balance storage handler --- pkg/processor/balance_storage_handler.go | 26 ++++++++++++++++++++++-- 1 file changed, 24 insertions(+), 2 deletions(-) diff --git a/pkg/processor/balance_storage_handler.go b/pkg/processor/balance_storage_handler.go index 4c054db0..d02baebe 100644 --- a/pkg/processor/balance_storage_handler.go +++ b/pkg/processor/balance_storage_handler.go @@ -16,23 +16,31 @@ package processor import ( "context" + "math/big" "github.com/coinbase/rosetta-cli/pkg/logger" "github.com/coinbase/rosetta-sdk-go/parser" "github.com/coinbase/rosetta-sdk-go/reconciler" + "github.com/coinbase/rosetta-sdk-go/storage/database" "github.com/coinbase/rosetta-sdk-go/storage/modules" "github.com/coinbase/rosetta-sdk-go/types" ) +const ( + totalAccountsReconciledCount = "totalAccountsReconciled" + totalAccountsSeenCount = "totalAccountsSeen" +) + var _ modules.BalanceStorageHandler = (*BalanceStorageHandler)(nil) // BalanceStorageHandler is invoked whenever a block is added // or removed from block storage so that balance changes // can be sent to other functions (ex: reconciler). type BalanceStorageHandler struct { - logger *logger.Logger - reconciler *reconciler.Reconciler + logger *logger.Logger + reconciler *reconciler.Reconciler + counterStorage *modules.CounterStorage reconcile bool interestingAccount *types.AccountCurrency @@ -42,12 +50,14 @@ type BalanceStorageHandler struct { func NewBalanceStorageHandler( logger *logger.Logger, reconciler *reconciler.Reconciler, + counterStorage *modules.CounterStorage, reconcile bool, interestingAccount *types.AccountCurrency, ) *BalanceStorageHandler { return &BalanceStorageHandler{ logger: logger, reconciler: reconciler, + counterStorage: counterStorage, reconcile: reconcile, interestingAccount: interestingAccount, } @@ -106,3 +116,15 @@ func (h *BalanceStorageHandler) BlockRemoved( // not removed return nil } + +// AccountsReconciled updates the total accounts reconciled by count. +func (h *BalanceStorageHandler) AccountsReconciled(ctx context.Context, dbTx database.Transaction, count int) error { + _, err := h.counterStorage.UpdateTransactional(ctx, dbTx, totalAccountsReconciledCount, big.NewInt(int64(count))) + return err +} + +// AccountsSeen updates the total accounts seen by count. +func (h *BalanceStorageHandler) AccountsSeen(ctx context.Context, dbTx database.Transaction, count int) error { + _, err := h.counterStorage.UpdateTransactional(ctx, dbTx, totalAccountsSeenCount, big.NewInt(int64(count))) + return err +} From c064bec5b724a8070d22cfda3074259a20911a47 Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Tue, 1 Dec 2020 13:00:51 -0600 Subject: [PATCH 205/376] update balance storage helper/handler --- go.mod | 2 +- go.sum | 2 ++ pkg/processor/balance_storage_handler.go | 9 ++------- pkg/processor/balance_storage_helper.go | 19 +++++++++++++++++-- pkg/processor/balance_storage_helper_test.go | 2 ++ 5 files changed, 24 insertions(+), 10 deletions(-) diff --git a/go.mod b/go.mod index 17bb610b..fe4bdab7 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/coinbase/rosetta-cli go 1.13 require ( - github.com/coinbase/rosetta-sdk-go v0.6.2-0.20201201181653-fa7c8e42b53e + github.com/coinbase/rosetta-sdk-go v0.6.2-0.20201201185830-8cd4faec860c github.com/fatih/color v1.10.0 github.com/olekukonko/tablewriter v0.0.2-0.20190409134802-7e037d187b0c github.com/spf13/cobra v1.1.1 diff --git a/go.sum b/go.sum index 22792f07..18d38990 100644 --- a/go.sum +++ b/go.sum @@ -85,6 +85,8 @@ github.com/coinbase/rosetta-sdk-go v0.6.0/go.mod h1:pqBibzTcpz0mLqb7is8fo2qf93tc github.com/coinbase/rosetta-sdk-go v0.6.1 h1:aOb5qstlX0uqP9HRC7wCY+YAZDzZbS2C/i3Qy/lR3xM= github.com/coinbase/rosetta-sdk-go v0.6.2-0.20201201181653-fa7c8e42b53e h1:OWe+e6cS7LvNpLDMabls5yiXEjNwqssYQDM8e0fuACY= github.com/coinbase/rosetta-sdk-go v0.6.2-0.20201201181653-fa7c8e42b53e/go.mod h1:MvQfsL2KlJ5786OdDviRIJE3agui2YcvS1CaQPDl1Yo= +github.com/coinbase/rosetta-sdk-go v0.6.2-0.20201201185830-8cd4faec860c h1:g9YLQ7Xlcbh4e7ehNghuUr0Tu7a0o+EKOV47q+3KjVg= +github.com/coinbase/rosetta-sdk-go v0.6.2-0.20201201185830-8cd4faec860c/go.mod h1:MvQfsL2KlJ5786OdDviRIJE3agui2YcvS1CaQPDl1Yo= github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= diff --git a/pkg/processor/balance_storage_handler.go b/pkg/processor/balance_storage_handler.go index d02baebe..6de2db22 100644 --- a/pkg/processor/balance_storage_handler.go +++ b/pkg/processor/balance_storage_handler.go @@ -27,11 +27,6 @@ import ( "github.com/coinbase/rosetta-sdk-go/types" ) -const ( - totalAccountsReconciledCount = "totalAccountsReconciled" - totalAccountsSeenCount = "totalAccountsSeen" -) - var _ modules.BalanceStorageHandler = (*BalanceStorageHandler)(nil) // BalanceStorageHandler is invoked whenever a block is added @@ -119,12 +114,12 @@ func (h *BalanceStorageHandler) BlockRemoved( // AccountsReconciled updates the total accounts reconciled by count. func (h *BalanceStorageHandler) AccountsReconciled(ctx context.Context, dbTx database.Transaction, count int) error { - _, err := h.counterStorage.UpdateTransactional(ctx, dbTx, totalAccountsReconciledCount, big.NewInt(int64(count))) + _, err := h.counterStorage.UpdateTransactional(ctx, dbTx, modules.ReconciledAccounts, big.NewInt(int64(count))) return err } // AccountsSeen updates the total accounts seen by count. func (h *BalanceStorageHandler) AccountsSeen(ctx context.Context, dbTx database.Transaction, count int) error { - _, err := h.counterStorage.UpdateTransactional(ctx, dbTx, totalAccountsSeenCount, big.NewInt(int64(count))) + _, err := h.counterStorage.UpdateTransactional(ctx, dbTx, modules.SeenAccounts, big.NewInt(int64(count))) return err } diff --git a/pkg/processor/balance_storage_helper.go b/pkg/processor/balance_storage_helper.go index 9648449d..b4d30195 100644 --- a/pkg/processor/balance_storage_helper.go +++ b/pkg/processor/balance_storage_helper.go @@ -17,10 +17,12 @@ package processor import ( "context" "fmt" + "math/big" "github.com/coinbase/rosetta-sdk-go/asserter" "github.com/coinbase/rosetta-sdk-go/fetcher" "github.com/coinbase/rosetta-sdk-go/parser" + "github.com/coinbase/rosetta-sdk-go/storage/database" "github.com/coinbase/rosetta-sdk-go/storage/modules" "github.com/coinbase/rosetta-sdk-go/syncer" "github.com/coinbase/rosetta-sdk-go/types" @@ -32,8 +34,9 @@ var _ modules.BalanceStorageHelper = (*BalanceStorageHelper)(nil) // BalanceStorageHelper implements the storage.Helper // interface. type BalanceStorageHelper struct { - network *types.NetworkIdentifier - fetcher *fetcher.Fetcher + network *types.NetworkIdentifier + fetcher *fetcher.Fetcher + counterStorage *modules.CounterStorage // Configuration settings lookupBalanceByBlock bool @@ -50,6 +53,7 @@ type BalanceStorageHelper struct { func NewBalanceStorageHelper( network *types.NetworkIdentifier, fetcher *fetcher.Fetcher, + counterStorage *modules.CounterStorage, lookupBalanceByBlock bool, exemptAccounts []*types.AccountCurrency, interestingOnly bool, @@ -67,6 +71,7 @@ func NewBalanceStorageHelper( return &BalanceStorageHelper{ network: network, fetcher: fetcher, + counterStorage: counterStorage, lookupBalanceByBlock: lookupBalanceByBlock, exemptAccounts: exemptMap, interestingAddresses: map[string]struct{}{}, @@ -154,3 +159,13 @@ func (h *BalanceStorageHelper) ExemptFunc() parser.ExemptOperation { func (h *BalanceStorageHelper) BalanceExemptions() []*types.BalanceExemption { return h.balanceExemptions } + +// AccountsReconciled returns the total accounts reconciled by count. +func (h *BalanceStorageHelper) AccountsReconciled(ctx context.Context, dbTx database.Transaction) (*big.Int, error) { + return h.counterStorage.GetTransactional(ctx, dbTx, modules.ReconciledAccounts) +} + +// AccountsSeen returns the total accounts seen by count. +func (h *BalanceStorageHelper) AccountsSeen(ctx context.Context, dbTx database.Transaction) (*big.Int, error) { + return h.counterStorage.GetTransactional(ctx, dbTx, modules.SeenAccounts) +} diff --git a/pkg/processor/balance_storage_helper_test.go b/pkg/processor/balance_storage_helper_test.go index 928efe4f..2e7cc5ee 100644 --- a/pkg/processor/balance_storage_helper_test.go +++ b/pkg/processor/balance_storage_helper_test.go @@ -78,6 +78,7 @@ func TestExemptFuncExemptAccounts(t *testing.T) { for name, test := range tests { t.Run(name, func(t *testing.T) { helper := NewBalanceStorageHelper( + nil, nil, nil, false, @@ -121,6 +122,7 @@ func TestExemptFuncInterestingParsing(t *testing.T) { for name, test := range tests { t.Run(name, func(t *testing.T) { helper := NewBalanceStorageHelper( + nil, nil, nil, false, From 4fb95ba2598cd76619efa0355aac75df8ee55d6a Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Tue, 1 Dec 2020 13:05:24 -0600 Subject: [PATCH 206/376] update pkg/tester/construction --- pkg/tester/construction.go | 46 +++++++++++++++++++++----------------- 1 file changed, 25 insertions(+), 21 deletions(-) diff --git a/pkg/tester/construction.go b/pkg/tester/construction.go index edf0b590..ade038f9 100644 --- a/pkg/tester/construction.go +++ b/pkg/tester/construction.go @@ -33,6 +33,8 @@ import ( "github.com/coinbase/rosetta-sdk-go/parser" "github.com/coinbase/rosetta-sdk-go/statefulsyncer" "github.com/coinbase/rosetta-sdk-go/storage/database" + storageErrs "github.com/coinbase/rosetta-sdk-go/storage/errors" + "github.com/coinbase/rosetta-sdk-go/storage/modules" "github.com/coinbase/rosetta-sdk-go/syncer" "github.com/coinbase/rosetta-sdk-go/types" "github.com/coinbase/rosetta-sdk-go/utils" @@ -54,15 +56,15 @@ var _ http.Handler = (*ConstructionTester)(nil) // ConstructionTester coordinates the `check:construction` test. type ConstructionTester struct { network *types.NetworkIdentifier - database storage.Database + database database.Database config *configuration.Configuration syncer *statefulsyncer.StatefulSyncer logger *logger.Logger onlineFetcher *fetcher.Fetcher - broadcastStorage *storage.BroadcastStorage - blockStorage *storage.BlockStorage - jobStorage *storage.JobStorage - counterStorage *storage.CounterStorage + broadcastStorage *modules.BroadcastStorage + blockStorage *modules.BlockStorage + jobStorage *modules.JobStorage + counterStorage *modules.CounterStorage coordinator *coordinator.Coordinator cancel context.CancelFunc signalReceived *bool @@ -84,15 +86,15 @@ func InitializeConstruction( log.Fatalf("%s: cannot create command path", err.Error()) } - opts := []storage.BadgerOption{} + opts := []database.BadgerOption{} if config.CompressionDisabled { - opts = append(opts, storage.WithoutCompression()) + opts = append(opts, database.WithoutCompression()) } if config.MemoryLimitDisabled { - opts = append(opts, storage.WithCustomSettings(storage.PerformanceBadgerOptions(dataPath))) + opts = append(opts, database.WithCustomSettings(database.PerformanceBadgerOptions(dataPath))) } - localStore, err := storage.NewBadgerStorage(ctx, dataPath, opts...) + localStore, err := database.NewBadgerDatabase(ctx, dataPath, opts...) if err != nil { log.Fatalf("%s: unable to initialize database", err.Error()) } @@ -107,7 +109,7 @@ func InitializeConstruction( log.Fatal("found balance exemptions but initial balance fetch disabled") } - counterStorage := storage.NewCounterStorage(localStore) + counterStorage := modules.NewCounterStorage(localStore) logger := logger.NewLogger( dataPath, false, @@ -116,15 +118,16 @@ func InitializeConstruction( false, ) - blockStorage := storage.NewBlockStorage(localStore) - keyStorage := storage.NewKeyStorage(localStore) + blockStorage := modules.NewBlockStorage(localStore) + keyStorage := modules.NewKeyStorage(localStore) coinStorageHelper := processor.NewCoinStorageHelper(blockStorage) - coinStorage := storage.NewCoinStorage(localStore, coinStorageHelper, onlineFetcher.Asserter) - balanceStorage := storage.NewBalanceStorage(localStore) + coinStorage := modules.NewCoinStorage(localStore, coinStorageHelper, onlineFetcher.Asserter) + balanceStorage := modules.NewBalanceStorage(localStore) balanceStorageHelper := processor.NewBalanceStorageHelper( network, onlineFetcher, + counterStorage, false, nil, true, @@ -135,13 +138,14 @@ func InitializeConstruction( balanceStorageHandler := processor.NewBalanceStorageHandler( logger, nil, + counterStorage, false, nil, ) balanceStorage.Initialize(balanceStorageHelper, balanceStorageHandler) - broadcastStorage := storage.NewBroadcastStorage( + broadcastStorage := modules.NewBroadcastStorage( localStore, config.Construction.StaleDepth, config.Construction.BroadcastLimit, @@ -209,7 +213,7 @@ func InitializeConstruction( return nil, fmt.Errorf("%w: unable to set coin balances", err) } - jobStorage := storage.NewJobStorage(localStore) + jobStorage := modules.NewJobStorage(localStore) coordinatorHelper := processor.NewCoordinatorHelper( offlineFetcher, onlineFetcher, @@ -255,10 +259,10 @@ func InitializeConstruction( counterStorage, logger, cancel, - []storage.BlockWorker{balanceStorage, coinStorage, broadcastStorage}, - syncer.DefaultCacheSize, - config.MaxSyncConcurrency, - config.MaxReorgDepth, + []modules.BlockWorker{counterStorage, balanceStorage, coinStorage, broadcastStorage}, + statefulsyncer.WithCacheSize(syncer.DefaultCacheSize), + statefulsyncer.WithMaxConcurrency(config.MaxSyncConcurrency), + statefulsyncer.WithPastBlockLimit(config.MaxReorgDepth), ) return &ConstructionTester{ @@ -360,7 +364,7 @@ func (t *ConstructionTester) StartSyncer( ) error { startIndex := int64(-1) _, err := t.blockStorage.GetHeadBlockIdentifier(ctx) - if errors.Is(err, storage.ErrHeadBlockNotFound) { + if errors.Is(err, storageErrs.ErrHeadBlockNotFound) { // If no head block exists, ensure we are at tip before starting. Otherwise, // we will unnecessarily sync tons of blocks before reaching any that matter. startIndex, err = t.waitForTip(ctx) From e76bcf6a80195000a2d7a36a9da67762c83d6549 Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Tue, 1 Dec 2020 13:10:53 -0600 Subject: [PATCH 207/376] update pkg --- pkg/tester/data.go | 66 +++++++++++++++++++++++++--------------------- 1 file changed, 36 insertions(+), 30 deletions(-) diff --git a/pkg/tester/data.go b/pkg/tester/data.go index 048fe1b5..accf474f 100644 --- a/pkg/tester/data.go +++ b/pkg/tester/data.go @@ -34,6 +34,8 @@ import ( "github.com/coinbase/rosetta-sdk-go/reconciler" "github.com/coinbase/rosetta-sdk-go/statefulsyncer" "github.com/coinbase/rosetta-sdk-go/storage/database" + storageErrs "github.com/coinbase/rosetta-sdk-go/storage/errors" + "github.com/coinbase/rosetta-sdk-go/storage/modules" "github.com/coinbase/rosetta-sdk-go/syncer" "github.com/coinbase/rosetta-sdk-go/types" "github.com/coinbase/rosetta-sdk-go/utils" @@ -71,14 +73,14 @@ var _ statefulsyncer.PruneHelper = (*DataTester)(nil) // DataTester coordinates the `check:data` test. type DataTester struct { network *types.NetworkIdentifier - database storage.Database + database database.Database config *configuration.Configuration syncer *statefulsyncer.StatefulSyncer reconciler *reconciler.Reconciler logger *logger.Logger - balanceStorage *storage.BalanceStorage - blockStorage *storage.BlockStorage - counterStorage *storage.CounterStorage + balanceStorage *modules.BalanceStorage + blockStorage *modules.BlockStorage + counterStorage *modules.CounterStorage reconcilerHandler *processor.ReconcilerHandler fetcher *fetcher.Fetcher signalReceived *bool @@ -148,15 +150,15 @@ func InitializeData( log.Fatalf("%s: cannot create command path", err.Error()) } - opts := []storage.BadgerOption{} + opts := []database.BadgerOption{} if config.CompressionDisabled { - opts = append(opts, storage.WithoutCompression()) + opts = append(opts, database.WithoutCompression()) } if config.MemoryLimitDisabled { - opts = append(opts, storage.WithCustomSettings(storage.PerformanceBadgerOptions(dataPath))) + opts = append(opts, database.WithCustomSettings(database.PerformanceBadgerOptions(dataPath))) } - localStore, err := storage.NewBadgerStorage(ctx, dataPath, opts...) + localStore, err := database.NewBadgerDatabase(ctx, dataPath, opts...) if err != nil { log.Fatalf("%s: unable to initialize database", err.Error()) } @@ -171,9 +173,9 @@ func InitializeData( log.Fatalf("%s: unable to load interesting accounts", err.Error()) } - counterStorage := storage.NewCounterStorage(localStore) - blockStorage := storage.NewBlockStorage(localStore) - balanceStorage := storage.NewBalanceStorage(localStore) + counterStorage := modules.NewCounterStorage(localStore) + blockStorage := modules.NewBlockStorage(localStore) + balanceStorage := modules.NewBalanceStorage(localStore) // Bootstrap balances, if provided. We need to do before initializing // the reconciler otherwise we won't reconcile bootstrapped accounts @@ -181,7 +183,7 @@ func InitializeData( if len(config.Data.BootstrapBalances) > 0 { _, err := blockStorage.GetHeadBlockIdentifier(ctx) switch { - case err == storage.ErrHeadBlockNotFound: + case err == storageErrs.ErrHeadBlockNotFound: err = balanceStorage.BootstrapBalances( ctx, config.Data.BootstrapBalances, @@ -275,11 +277,12 @@ func InitializeData( rOpts..., ) - blockWorkers := []storage.BlockWorker{} + blockWorkers := []modules.BlockWorker{counterStorage} if !config.Data.BalanceTrackingDisabled { balanceStorageHelper := processor.NewBalanceStorageHelper( network, fetcher, + counterStorage, historicalBalanceEnabled, exemptAccounts, false, @@ -290,6 +293,7 @@ func InitializeData( balanceStorageHandler := processor.NewBalanceStorageHandler( logger, r, + counterStorage, shouldReconcile(config), interestingAccount, ) @@ -301,7 +305,7 @@ func InitializeData( if !config.Data.CoinTrackingDisabled { coinStorageHelper := processor.NewCoinStorageHelper(blockStorage) - coinStorage := storage.NewCoinStorage(localStore, coinStorageHelper, fetcher.Asserter) + coinStorage := modules.NewCoinStorage(localStore, coinStorageHelper, fetcher.Asserter) blockWorkers = append(blockWorkers, coinStorage) } @@ -315,9 +319,9 @@ func InitializeData( logger, cancel, blockWorkers, - syncer.DefaultCacheSize, - config.MaxSyncConcurrency, - config.MaxReorgDepth, + statefulsyncer.WithCacheSize(syncer.DefaultCacheSize), + statefulsyncer.WithMaxConcurrency(config.MaxSyncConcurrency), + statefulsyncer.WithPastBlockLimit(config.MaxReorgDepth), ) return &DataTester{ @@ -512,7 +516,7 @@ func (t *DataTester) EndReconciliationCoverage( // nolint:gocognit case <-tc.C: headBlock, err := t.blockStorage.GetBlock(ctx, nil) - if errors.Is(err, storage.ErrHeadBlockNotFound) { + if errors.Is(err, storageErrs.ErrHeadBlockNotFound) { continue } if err != nil { @@ -665,22 +669,22 @@ func (t *DataTester) WatchEndConditions( // CompleteReconciliations returns the sum of all failed, exempt, and successful // reconciliations. func (t *DataTester) CompleteReconciliations(ctx context.Context) (int64, error) { - activeReconciliations, err := t.counterStorage.Get(ctx, storage.ActiveReconciliationCounter) + activeReconciliations, err := t.counterStorage.Get(ctx, modules.ActiveReconciliationCounter) if err != nil { return -1, fmt.Errorf("%w: cannot get active reconciliations counter", err) } - exemptReconciliations, err := t.counterStorage.Get(ctx, storage.ExemptReconciliationCounter) + exemptReconciliations, err := t.counterStorage.Get(ctx, modules.ExemptReconciliationCounter) if err != nil { return -1, fmt.Errorf("%w: cannot get exempt reconciliations counter", err) } - failedReconciliations, err := t.counterStorage.Get(ctx, storage.FailedReconciliationCounter) + failedReconciliations, err := t.counterStorage.Get(ctx, modules.FailedReconciliationCounter) if err != nil { return -1, fmt.Errorf("%w: cannot get failed reconciliations counter", err) } - skippedReconciliations, err := t.counterStorage.Get(ctx, storage.SkippedReconciliationsCounter) + skippedReconciliations, err := t.counterStorage.Get(ctx, modules.SkippedReconciliationsCounter) if err != nil { return -1, fmt.Errorf("%w: cannot get skipped reconciliations counter", err) } @@ -952,14 +956,14 @@ func (t *DataTester) recursiveOpSearch( } defer utils.RemoveTempDir(tmpDir) - localStore, err := storage.NewBadgerStorage(ctx, tmpDir) + localStore, err := database.NewBadgerDatabase(ctx, tmpDir) if err != nil { return nil, fmt.Errorf("%w: unable to initialize database", err) } - counterStorage := storage.NewCounterStorage(localStore) - blockStorage := storage.NewBlockStorage(localStore) - balanceStorage := storage.NewBalanceStorage(localStore) + counterStorage := modules.NewCounterStorage(localStore) + blockStorage := modules.NewBlockStorage(localStore) + balanceStorage := modules.NewBalanceStorage(localStore) logger := logger.NewLogger( tmpDir, @@ -1005,6 +1009,7 @@ func (t *DataTester) recursiveOpSearch( balanceStorageHelper := processor.NewBalanceStorageHelper( t.network, t.fetcher, + counterStorage, t.historicalBalanceEnabled, nil, false, @@ -1015,6 +1020,7 @@ func (t *DataTester) recursiveOpSearch( balanceStorageHandler := processor.NewBalanceStorageHandler( logger, r, + counterStorage, true, accountCurrency, ) @@ -1029,10 +1035,10 @@ func (t *DataTester) recursiveOpSearch( counterStorage, logger, cancel, - []storage.BlockWorker{balanceStorage}, - syncer.DefaultCacheSize, - t.config.MaxSyncConcurrency, - t.config.MaxReorgDepth, + []modules.BlockWorker{balanceStorage}, + statefulsyncer.WithCacheSize(syncer.DefaultCacheSize), + statefulsyncer.WithMaxConcurrency(t.config.MaxSyncConcurrency), + statefulsyncer.WithPastBlockLimit(t.config.MaxReorgDepth), ) g, ctx := errgroup.WithContext(ctx) From d00b1d5794ca119ef039b8830f56b6c9796e828a Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Tue, 1 Dec 2020 13:12:05 -0600 Subject: [PATCH 208/376] finish breaking changes --- cmd/utils_train_zstd.go | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/cmd/utils_train_zstd.go b/cmd/utils_train_zstd.go index d3cff75e..5a338f00 100644 --- a/cmd/utils_train_zstd.go +++ b/cmd/utils_train_zstd.go @@ -21,6 +21,7 @@ import ( "strconv" "github.com/coinbase/rosetta-sdk-go/storage/database" + "github.com/coinbase/rosetta-sdk-go/storage/encoder" "github.com/fatih/color" "github.com/spf13/cobra" ) @@ -58,9 +59,9 @@ func runTrainZstdCmd(cmd *cobra.Command, args []string) error { return fmt.Errorf("%w: unable to convert max items to integer", err) } - compressorEntries := []*storage.CompressorEntry{} + compressorEntries := []*encoder.CompressorEntry{} if len(args) > trainArgs { - compressorEntries = append(compressorEntries, &storage.CompressorEntry{ + compressorEntries = append(compressorEntries, &encoder.CompressorEntry{ Namespace: namespace, DictionaryPath: args[4], }) @@ -70,7 +71,7 @@ func runTrainZstdCmd(cmd *cobra.Command, args []string) error { log.Printf("Running zstd training (this could take a while)...") - _, _, err = storage.BadgerTrain( + _, _, err = database.BadgerTrain( Context, namespace, databasePath, From 96e49c2cfdcd55007e0d1ae395bf405fca86fb70 Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Tue, 1 Dec 2020 13:14:15 -0600 Subject: [PATCH 209/376] use estimated reconciliation coverage --- pkg/results/data_results.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/results/data_results.go b/pkg/results/data_results.go index 5935e524..1afdcdd1 100644 --- a/pkg/results/data_results.go +++ b/pkg/results/data_results.go @@ -250,7 +250,7 @@ func ComputeCheckDataStats( } if balances != nil { - coverage, err := balances.ReconciliationCoverage(ctx, 0) + coverage, err := balances.EstimatedReconciliationCoverage(ctx) if err != nil { log.Printf("%s: cannot get reconcile coverage", err.Error()) return nil From ef8395cd57bb14b739af5a3cf5afa7253e171ca8 Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Tue, 1 Dec 2020 13:18:45 -0600 Subject: [PATCH 210/376] remove unnecessary mocks --- Makefile | 5 - go.sum | 9 - mocks/constructor/handler.go | 43 ---- mocks/constructor/helper.go | 441 ----------------------------------- 4 files changed, 498 deletions(-) delete mode 100644 mocks/constructor/handler.go delete mode 100644 mocks/constructor/helper.go diff --git a/Makefile b/Makefile index 84e8fc85..22429225 100644 --- a/Makefile +++ b/Makefile @@ -63,8 +63,3 @@ build: install: go install ./... - -mocks: - rm -rf mocks; - mockery --dir pkg/constructor --all --case underscore --outpkg constructor --output mocks/constructor; - ${ADDLICENCE_SCRIPT} .; diff --git a/go.sum b/go.sum index 18d38990..1048607a 100644 --- a/go.sum +++ b/go.sum @@ -80,11 +80,6 @@ github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+ github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cloudflare/cloudflare-go v0.10.2-0.20190916151808-a80f83b9add9/go.mod h1:1MxXX1Ux4x6mqPmjkUgTP1CdXIBXKX7T+Jk9Gxrmx+U= -github.com/coinbase/rosetta-sdk-go v0.6.0 h1:U8/NhkPo7CGJR8Ud82Y0HqtujXSVzGZKscmxOHsmS54= -github.com/coinbase/rosetta-sdk-go v0.6.0/go.mod h1:pqBibzTcpz0mLqb7is8fo2qf93tcHjItJ3u7Yud8RI4= -github.com/coinbase/rosetta-sdk-go v0.6.1 h1:aOb5qstlX0uqP9HRC7wCY+YAZDzZbS2C/i3Qy/lR3xM= -github.com/coinbase/rosetta-sdk-go v0.6.2-0.20201201181653-fa7c8e42b53e h1:OWe+e6cS7LvNpLDMabls5yiXEjNwqssYQDM8e0fuACY= -github.com/coinbase/rosetta-sdk-go v0.6.2-0.20201201181653-fa7c8e42b53e/go.mod h1:MvQfsL2KlJ5786OdDviRIJE3agui2YcvS1CaQPDl1Yo= github.com/coinbase/rosetta-sdk-go v0.6.2-0.20201201185830-8cd4faec860c h1:g9YLQ7Xlcbh4e7ehNghuUr0Tu7a0o+EKOV47q+3KjVg= github.com/coinbase/rosetta-sdk-go v0.6.2-0.20201201185830-8cd4faec860c/go.mod h1:MvQfsL2KlJ5786OdDviRIJE3agui2YcvS1CaQPDl1Yo= github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= @@ -124,8 +119,6 @@ github.com/dvyukov/go-fuzz v0.0.0-20200318091601-be3528f3a813/go.mod h1:11Gm+ccJ github.com/edsrzf/mmap-go v0.0.0-20160512033002-935e0e8a636c/go.mod h1:YO35OhQPt3KJa3ryjFM5Bs14WD66h8eGKpfaBNrHW5M= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= -github.com/ethereum/go-ethereum v1.9.23 h1:SIKhg/z4Q7AbvqcxuPYvMxf36che/Rq/Pp0IdYEkbtw= -github.com/ethereum/go-ethereum v1.9.23/go.mod h1:JIfVb6esrqALTExdz9hRYvrP0xBDf6wCncIu1hNwHpM= github.com/ethereum/go-ethereum v1.9.24 h1:6AK+ORt3EMDO+FTjzXy/AQwHMbu52J2nYHIjyQX9azQ= github.com/ethereum/go-ethereum v1.9.24/go.mod h1:JIfVb6esrqALTExdz9hRYvrP0xBDf6wCncIu1hNwHpM= github.com/fatih/color v1.3.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= @@ -387,8 +380,6 @@ github.com/tyler-smith/go-bip39 v1.0.1-0.20181017060643-dbb3b84ba2ef/go.mod h1:s github.com/tyler-smith/go-bip39 v1.0.2/go.mod h1:sJ5fKU0s6JVwZjjcUEX2zFOnvq0ASQ2K9Zr6cf67kNs= github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= -github.com/vmihailenco/msgpack/v5 v5.0.0-beta.9 h1:iBRIniTnWOo0kqkg3k3XR8Vn6OCkVlIuZNo0UoBrKx4= -github.com/vmihailenco/msgpack/v5 v5.0.0-beta.9/go.mod h1:HVxBVPUK/+fZMonk4bi1islLa8V3cfnBug0+4dykPzo= github.com/vmihailenco/msgpack/v5 v5.1.0 h1:+od5YbEXxW95SPlW6beocmt8nOtlh83zqat5Ip9Hwdc= github.com/vmihailenco/msgpack/v5 v5.1.0/go.mod h1:C5gboKD0TJPqWDTVTtrQNfRbiBwHZGo8UTqP/9/XvLI= github.com/vmihailenco/tagparser v0.1.2 h1:gnjoVuB/kljJ5wICEEOpx98oXMWPLj22G67Vbd1qPqc= diff --git a/mocks/constructor/handler.go b/mocks/constructor/handler.go deleted file mode 100644 index 4afe30ef..00000000 --- a/mocks/constructor/handler.go +++ /dev/null @@ -1,43 +0,0 @@ -// Code generated by mockery v1.0.0. DO NOT EDIT. - -package constructor - -import ( - context "context" - - types "github.com/coinbase/rosetta-sdk-go/types" - mock "github.com/stretchr/testify/mock" -) - -// Handler is an autogenerated mock type for the Handler type -type Handler struct { - mock.Mock -} - -// AddressCreated provides a mock function with given fields: _a0, _a1 -func (_m *Handler) AddressCreated(_a0 context.Context, _a1 string) error { - ret := _m.Called(_a0, _a1) - - var r0 error - if rf, ok := ret.Get(0).(func(context.Context, string) error); ok { - r0 = rf(_a0, _a1) - } else { - r0 = ret.Error(0) - } - - return r0 -} - -// TransactionCreated provides a mock function with given fields: _a0, _a1, _a2 -func (_m *Handler) TransactionCreated(_a0 context.Context, _a1 string, _a2 *types.TransactionIdentifier) error { - ret := _m.Called(_a0, _a1, _a2) - - var r0 error - if rf, ok := ret.Get(0).(func(context.Context, string, *types.TransactionIdentifier) error); ok { - r0 = rf(_a0, _a1, _a2) - } else { - r0 = ret.Error(0) - } - - return r0 -} diff --git a/mocks/constructor/helper.go b/mocks/constructor/helper.go deleted file mode 100644 index 91eab4a2..00000000 --- a/mocks/constructor/helper.go +++ /dev/null @@ -1,441 +0,0 @@ -// Code generated by mockery v1.0.0. DO NOT EDIT. - -package constructor - -import ( - big "math/big" - - context "context" - - keys "github.com/coinbase/rosetta-sdk-go/keys" - - mock "github.com/stretchr/testify/mock" - - storage "github.com/coinbase/rosetta-sdk-go/storage/database" - - types "github.com/coinbase/rosetta-sdk-go/types" -) - -// Helper is an autogenerated mock type for the Helper type -type Helper struct { - mock.Mock -} - -// AccountBalance provides a mock function with given fields: _a0, _a1, _a2 -func (_m *Helper) AccountBalance(_a0 context.Context, _a1 *types.AccountIdentifier, _a2 *types.Currency) (*big.Int, error) { - ret := _m.Called(_a0, _a1, _a2) - - var r0 *big.Int - if rf, ok := ret.Get(0).(func(context.Context, *types.AccountIdentifier, *types.Currency) *big.Int); ok { - r0 = rf(_a0, _a1, _a2) - } else { - if ret.Get(0) != nil { - r0 = ret.Get(0).(*big.Int) - } - } - - var r1 error - if rf, ok := ret.Get(1).(func(context.Context, *types.AccountIdentifier, *types.Currency) error); ok { - r1 = rf(_a0, _a1, _a2) - } else { - r1 = ret.Error(1) - } - - return r0, r1 -} - -// AllAddresses provides a mock function with given fields: ctx -func (_m *Helper) AllAddresses(ctx context.Context) ([]string, error) { - ret := _m.Called(ctx) - - var r0 []string - if rf, ok := ret.Get(0).(func(context.Context) []string); ok { - r0 = rf(ctx) - } else { - if ret.Get(0) != nil { - r0 = ret.Get(0).([]string) - } - } - - var r1 error - if rf, ok := ret.Get(1).(func(context.Context) error); ok { - r1 = rf(ctx) - } else { - r1 = ret.Error(1) - } - - return r0, r1 -} - -// AllBroadcasts provides a mock function with given fields: ctx -func (_m *Helper) AllBroadcasts(ctx context.Context) ([]*storage.Broadcast, error) { - ret := _m.Called(ctx) - - var r0 []*storage.Broadcast - if rf, ok := ret.Get(0).(func(context.Context) []*storage.Broadcast); ok { - r0 = rf(ctx) - } else { - if ret.Get(0) != nil { - r0 = ret.Get(0).([]*storage.Broadcast) - } - } - - var r1 error - if rf, ok := ret.Get(1).(func(context.Context) error); ok { - r1 = rf(ctx) - } else { - r1 = ret.Error(1) - } - - return r0, r1 -} - -// Broadcast provides a mock function with given fields: _a0, _a1, _a2, _a3, _a4 -func (_m *Helper) Broadcast(_a0 context.Context, _a1 string, _a2 []*types.Operation, _a3 *types.TransactionIdentifier, _a4 string) error { - ret := _m.Called(_a0, _a1, _a2, _a3, _a4) - - var r0 error - if rf, ok := ret.Get(0).(func(context.Context, string, []*types.Operation, *types.TransactionIdentifier, string) error); ok { - r0 = rf(_a0, _a1, _a2, _a3, _a4) - } else { - r0 = ret.Error(0) - } - - return r0 -} - -// ClearBroadcasts provides a mock function with given fields: ctx -func (_m *Helper) ClearBroadcasts(ctx context.Context) ([]*storage.Broadcast, error) { - ret := _m.Called(ctx) - - var r0 []*storage.Broadcast - if rf, ok := ret.Get(0).(func(context.Context) []*storage.Broadcast); ok { - r0 = rf(ctx) - } else { - if ret.Get(0) != nil { - r0 = ret.Get(0).([]*storage.Broadcast) - } - } - - var r1 error - if rf, ok := ret.Get(1).(func(context.Context) error); ok { - r1 = rf(ctx) - } else { - r1 = ret.Error(1) - } - - return r0, r1 -} - -// CoinBalance provides a mock function with given fields: _a0, _a1, _a2 -func (_m *Helper) CoinBalance(_a0 context.Context, _a1 *types.AccountIdentifier, _a2 *types.Currency) (*big.Int, *types.CoinIdentifier, error) { - ret := _m.Called(_a0, _a1, _a2) - - var r0 *big.Int - if rf, ok := ret.Get(0).(func(context.Context, *types.AccountIdentifier, *types.Currency) *big.Int); ok { - r0 = rf(_a0, _a1, _a2) - } else { - if ret.Get(0) != nil { - r0 = ret.Get(0).(*big.Int) - } - } - - var r1 *types.CoinIdentifier - if rf, ok := ret.Get(1).(func(context.Context, *types.AccountIdentifier, *types.Currency) *types.CoinIdentifier); ok { - r1 = rf(_a0, _a1, _a2) - } else { - if ret.Get(1) != nil { - r1 = ret.Get(1).(*types.CoinIdentifier) - } - } - - var r2 error - if rf, ok := ret.Get(2).(func(context.Context, *types.AccountIdentifier, *types.Currency) error); ok { - r2 = rf(_a0, _a1, _a2) - } else { - r2 = ret.Error(2) - } - - return r0, r1, r2 -} - -// Combine provides a mock function with given fields: _a0, _a1, _a2, _a3 -func (_m *Helper) Combine(_a0 context.Context, _a1 *types.NetworkIdentifier, _a2 string, _a3 []*types.Signature) (string, error) { - ret := _m.Called(_a0, _a1, _a2, _a3) - - var r0 string - if rf, ok := ret.Get(0).(func(context.Context, *types.NetworkIdentifier, string, []*types.Signature) string); ok { - r0 = rf(_a0, _a1, _a2, _a3) - } else { - r0 = ret.Get(0).(string) - } - - var r1 error - if rf, ok := ret.Get(1).(func(context.Context, *types.NetworkIdentifier, string, []*types.Signature) error); ok { - r1 = rf(_a0, _a1, _a2, _a3) - } else { - r1 = ret.Error(1) - } - - return r0, r1 -} - -// Derive provides a mock function with given fields: _a0, _a1, _a2, _a3 -func (_m *Helper) Derive(_a0 context.Context, _a1 *types.NetworkIdentifier, _a2 *types.PublicKey, _a3 map[string]interface{}) (string, map[string]interface{}, error) { - ret := _m.Called(_a0, _a1, _a2, _a3) - - var r0 string - if rf, ok := ret.Get(0).(func(context.Context, *types.NetworkIdentifier, *types.PublicKey, map[string]interface{}) string); ok { - r0 = rf(_a0, _a1, _a2, _a3) - } else { - r0 = ret.Get(0).(string) - } - - var r1 map[string]interface{} - if rf, ok := ret.Get(1).(func(context.Context, *types.NetworkIdentifier, *types.PublicKey, map[string]interface{}) map[string]interface{}); ok { - r1 = rf(_a0, _a1, _a2, _a3) - } else { - if ret.Get(1) != nil { - r1 = ret.Get(1).(map[string]interface{}) - } - } - - var r2 error - if rf, ok := ret.Get(2).(func(context.Context, *types.NetworkIdentifier, *types.PublicKey, map[string]interface{}) error); ok { - r2 = rf(_a0, _a1, _a2, _a3) - } else { - r2 = ret.Error(2) - } - - return r0, r1, r2 -} - -// Hash provides a mock function with given fields: _a0, _a1, _a2 -func (_m *Helper) Hash(_a0 context.Context, _a1 *types.NetworkIdentifier, _a2 string) (*types.TransactionIdentifier, error) { - ret := _m.Called(_a0, _a1, _a2) - - var r0 *types.TransactionIdentifier - if rf, ok := ret.Get(0).(func(context.Context, *types.NetworkIdentifier, string) *types.TransactionIdentifier); ok { - r0 = rf(_a0, _a1, _a2) - } else { - if ret.Get(0) != nil { - r0 = ret.Get(0).(*types.TransactionIdentifier) - } - } - - var r1 error - if rf, ok := ret.Get(1).(func(context.Context, *types.NetworkIdentifier, string) error); ok { - r1 = rf(_a0, _a1, _a2) - } else { - r1 = ret.Error(1) - } - - return r0, r1 -} - -// HeadBlockExists provides a mock function with given fields: _a0 -func (_m *Helper) HeadBlockExists(_a0 context.Context) bool { - ret := _m.Called(_a0) - - var r0 bool - if rf, ok := ret.Get(0).(func(context.Context) bool); ok { - r0 = rf(_a0) - } else { - r0 = ret.Get(0).(bool) - } - - return r0 -} - -// LockedAddresses provides a mock function with given fields: _a0 -func (_m *Helper) LockedAddresses(_a0 context.Context) ([]string, error) { - ret := _m.Called(_a0) - - var r0 []string - if rf, ok := ret.Get(0).(func(context.Context) []string); ok { - r0 = rf(_a0) - } else { - if ret.Get(0) != nil { - r0 = ret.Get(0).([]string) - } - } - - var r1 error - if rf, ok := ret.Get(1).(func(context.Context) error); ok { - r1 = rf(_a0) - } else { - r1 = ret.Error(1) - } - - return r0, r1 -} - -// Metadata provides a mock function with given fields: _a0, _a1, _a2 -func (_m *Helper) Metadata(_a0 context.Context, _a1 *types.NetworkIdentifier, _a2 map[string]interface{}) (map[string]interface{}, error) { - ret := _m.Called(_a0, _a1, _a2) - - var r0 map[string]interface{} - if rf, ok := ret.Get(0).(func(context.Context, *types.NetworkIdentifier, map[string]interface{}) map[string]interface{}); ok { - r0 = rf(_a0, _a1, _a2) - } else { - if ret.Get(0) != nil { - r0 = ret.Get(0).(map[string]interface{}) - } - } - - var r1 error - if rf, ok := ret.Get(1).(func(context.Context, *types.NetworkIdentifier, map[string]interface{}) error); ok { - r1 = rf(_a0, _a1, _a2) - } else { - r1 = ret.Error(1) - } - - return r0, r1 -} - -// Parse provides a mock function with given fields: _a0, _a1, _a2, _a3 -func (_m *Helper) Parse(_a0 context.Context, _a1 *types.NetworkIdentifier, _a2 bool, _a3 string) ([]*types.Operation, []string, map[string]interface{}, error) { - ret := _m.Called(_a0, _a1, _a2, _a3) - - var r0 []*types.Operation - if rf, ok := ret.Get(0).(func(context.Context, *types.NetworkIdentifier, bool, string) []*types.Operation); ok { - r0 = rf(_a0, _a1, _a2, _a3) - } else { - if ret.Get(0) != nil { - r0 = ret.Get(0).([]*types.Operation) - } - } - - var r1 []string - if rf, ok := ret.Get(1).(func(context.Context, *types.NetworkIdentifier, bool, string) []string); ok { - r1 = rf(_a0, _a1, _a2, _a3) - } else { - if ret.Get(1) != nil { - r1 = ret.Get(1).([]string) - } - } - - var r2 map[string]interface{} - if rf, ok := ret.Get(2).(func(context.Context, *types.NetworkIdentifier, bool, string) map[string]interface{}); ok { - r2 = rf(_a0, _a1, _a2, _a3) - } else { - if ret.Get(2) != nil { - r2 = ret.Get(2).(map[string]interface{}) - } - } - - var r3 error - if rf, ok := ret.Get(3).(func(context.Context, *types.NetworkIdentifier, bool, string) error); ok { - r3 = rf(_a0, _a1, _a2, _a3) - } else { - r3 = ret.Error(3) - } - - return r0, r1, r2, r3 -} - -// Payloads provides a mock function with given fields: _a0, _a1, _a2, _a3 -func (_m *Helper) Payloads(_a0 context.Context, _a1 *types.NetworkIdentifier, _a2 []*types.Operation, _a3 map[string]interface{}) (string, []*types.SigningPayload, error) { - ret := _m.Called(_a0, _a1, _a2, _a3) - - var r0 string - if rf, ok := ret.Get(0).(func(context.Context, *types.NetworkIdentifier, []*types.Operation, map[string]interface{}) string); ok { - r0 = rf(_a0, _a1, _a2, _a3) - } else { - r0 = ret.Get(0).(string) - } - - var r1 []*types.SigningPayload - if rf, ok := ret.Get(1).(func(context.Context, *types.NetworkIdentifier, []*types.Operation, map[string]interface{}) []*types.SigningPayload); ok { - r1 = rf(_a0, _a1, _a2, _a3) - } else { - if ret.Get(1) != nil { - r1 = ret.Get(1).([]*types.SigningPayload) - } - } - - var r2 error - if rf, ok := ret.Get(2).(func(context.Context, *types.NetworkIdentifier, []*types.Operation, map[string]interface{}) error); ok { - r2 = rf(_a0, _a1, _a2, _a3) - } else { - r2 = ret.Error(2) - } - - return r0, r1, r2 -} - -// Preprocess provides a mock function with given fields: _a0, _a1, _a2, _a3 -func (_m *Helper) Preprocess(_a0 context.Context, _a1 *types.NetworkIdentifier, _a2 []*types.Operation, _a3 map[string]interface{}) (map[string]interface{}, error) { - ret := _m.Called(_a0, _a1, _a2, _a3) - - var r0 map[string]interface{} - if rf, ok := ret.Get(0).(func(context.Context, *types.NetworkIdentifier, []*types.Operation, map[string]interface{}) map[string]interface{}); ok { - r0 = rf(_a0, _a1, _a2, _a3) - } else { - if ret.Get(0) != nil { - r0 = ret.Get(0).(map[string]interface{}) - } - } - - var r1 error - if rf, ok := ret.Get(1).(func(context.Context, *types.NetworkIdentifier, []*types.Operation, map[string]interface{}) error); ok { - r1 = rf(_a0, _a1, _a2, _a3) - } else { - r1 = ret.Error(1) - } - - return r0, r1 -} - -// RandomAmount provides a mock function with given fields: _a0, _a1 -func (_m *Helper) RandomAmount(_a0 *big.Int, _a1 *big.Int) *big.Int { - ret := _m.Called(_a0, _a1) - - var r0 *big.Int - if rf, ok := ret.Get(0).(func(*big.Int, *big.Int) *big.Int); ok { - r0 = rf(_a0, _a1) - } else { - if ret.Get(0) != nil { - r0 = ret.Get(0).(*big.Int) - } - } - - return r0 -} - -// Sign provides a mock function with given fields: _a0, _a1 -func (_m *Helper) Sign(_a0 context.Context, _a1 []*types.SigningPayload) ([]*types.Signature, error) { - ret := _m.Called(_a0, _a1) - - var r0 []*types.Signature - if rf, ok := ret.Get(0).(func(context.Context, []*types.SigningPayload) []*types.Signature); ok { - r0 = rf(_a0, _a1) - } else { - if ret.Get(0) != nil { - r0 = ret.Get(0).([]*types.Signature) - } - } - - var r1 error - if rf, ok := ret.Get(1).(func(context.Context, []*types.SigningPayload) error); ok { - r1 = rf(_a0, _a1) - } else { - r1 = ret.Error(1) - } - - return r0, r1 -} - -// StoreKey provides a mock function with given fields: _a0, _a1, _a2 -func (_m *Helper) StoreKey(_a0 context.Context, _a1 string, _a2 *keys.KeyPair) error { - ret := _m.Called(_a0, _a1, _a2) - - var r0 error - if rf, ok := ret.Get(0).(func(context.Context, string, *keys.KeyPair) error); ok { - r0 = rf(_a0, _a1, _a2) - } else { - r0 = ret.Error(0) - } - - return r0 -} From 036c55b248f531d723cd95849ba5e481c977d264 Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Tue, 1 Dec 2020 13:59:10 -0600 Subject: [PATCH 211/376] Configurable pruning frequency --- configuration/types.go | 5 +++++ go.mod | 2 +- go.sum | 6 ++++++ pkg/tester/data.go | 13 ++++++++++--- 4 files changed, 22 insertions(+), 4 deletions(-) diff --git a/configuration/types.go b/configuration/types.go index 9da2c1fd..5a25f670 100644 --- a/configuration/types.go +++ b/configuration/types.go @@ -315,6 +315,11 @@ type DataConfiguration struct { // previously synced block. PruningDisabled bool `json:"pruning_disabled"` + // PruningFrequency is the frequency (in seconds) that we attempt + // to prune blocks. If not populated, we use the default value + // provided in the `statefulsyncer` package. + PruningFrequency *int `json:"pruning_frequency,omitempty"` + // InitialBalanceFetchDisabled configures rosetta-cli // not to lookup the balance of newly seen accounts at the // parent block before applying operations. Disabling diff --git a/go.mod b/go.mod index fe4bdab7..26485918 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/coinbase/rosetta-cli go 1.13 require ( - github.com/coinbase/rosetta-sdk-go v0.6.2-0.20201201185830-8cd4faec860c + github.com/coinbase/rosetta-sdk-go v0.6.2-0.20201201194354-4e3d0f143850 github.com/fatih/color v1.10.0 github.com/olekukonko/tablewriter v0.0.2-0.20190409134802-7e037d187b0c github.com/spf13/cobra v1.1.1 diff --git a/go.sum b/go.sum index 1048607a..0488f32b 100644 --- a/go.sum +++ b/go.sum @@ -82,6 +82,12 @@ github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDk github.com/cloudflare/cloudflare-go v0.10.2-0.20190916151808-a80f83b9add9/go.mod h1:1MxXX1Ux4x6mqPmjkUgTP1CdXIBXKX7T+Jk9Gxrmx+U= github.com/coinbase/rosetta-sdk-go v0.6.2-0.20201201185830-8cd4faec860c h1:g9YLQ7Xlcbh4e7ehNghuUr0Tu7a0o+EKOV47q+3KjVg= github.com/coinbase/rosetta-sdk-go v0.6.2-0.20201201185830-8cd4faec860c/go.mod h1:MvQfsL2KlJ5786OdDviRIJE3agui2YcvS1CaQPDl1Yo= +github.com/coinbase/rosetta-sdk-go v0.6.2-0.20201201193353-a11b3f3310c3 h1:wLrRp4gjVsX1/L3okNpXiaK5osCzeAl+h2XMocPYYxk= +github.com/coinbase/rosetta-sdk-go v0.6.2-0.20201201193353-a11b3f3310c3/go.mod h1:MvQfsL2KlJ5786OdDviRIJE3agui2YcvS1CaQPDl1Yo= +github.com/coinbase/rosetta-sdk-go v0.6.2-0.20201201193814-f4843ede4a3f h1:ku6sf059Al5duyD2a4/vV3p9yvGW1R/eMO3Cu67qPuM= +github.com/coinbase/rosetta-sdk-go v0.6.2-0.20201201193814-f4843ede4a3f/go.mod h1:MvQfsL2KlJ5786OdDviRIJE3agui2YcvS1CaQPDl1Yo= +github.com/coinbase/rosetta-sdk-go v0.6.2-0.20201201194354-4e3d0f143850 h1:2X4q0lyGzELPolyYFlSqE4bg+NCVyCT4+Mud2UxNI9Q= +github.com/coinbase/rosetta-sdk-go v0.6.2-0.20201201194354-4e3d0f143850/go.mod h1:MvQfsL2KlJ5786OdDviRIJE3agui2YcvS1CaQPDl1Yo= github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= diff --git a/pkg/tester/data.go b/pkg/tester/data.go index accf474f..19b17d42 100644 --- a/pkg/tester/data.go +++ b/pkg/tester/data.go @@ -310,6 +310,15 @@ func InitializeData( blockWorkers = append(blockWorkers, coinStorage) } + statefulSyncerOptions := []statefulsyncer.Option{ + statefulsyncer.WithCacheSize(syncer.DefaultCacheSize), + statefulsyncer.WithMaxConcurrency(config.MaxSyncConcurrency), + statefulsyncer.WithPastBlockLimit(config.MaxReorgDepth), + } + if config.Data.PruningFrequency != nil { + statefulSyncerOptions = append(statefulSyncerOptions, statefulsyncer.WithPruneSleepTime(*config.Data.PruningFrequency)) + } + syncer := statefulsyncer.New( ctx, network, @@ -319,9 +328,7 @@ func InitializeData( logger, cancel, blockWorkers, - statefulsyncer.WithCacheSize(syncer.DefaultCacheSize), - statefulsyncer.WithMaxConcurrency(config.MaxSyncConcurrency), - statefulsyncer.WithPastBlockLimit(config.MaxReorgDepth), + statefulSyncerOptions..., ) return &DataTester{ From 31205fb345342c93941ba55f6fbb87a2ee1fb1e6 Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Tue, 1 Dec 2020 17:54:21 -0600 Subject: [PATCH 212/376] update to official release --- go.mod | 2 +- go.sum | 10 ++-------- 2 files changed, 3 insertions(+), 9 deletions(-) diff --git a/go.mod b/go.mod index 26485918..b19152f6 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/coinbase/rosetta-cli go 1.13 require ( - github.com/coinbase/rosetta-sdk-go v0.6.2-0.20201201194354-4e3d0f143850 + github.com/coinbase/rosetta-sdk-go v0.6.2 github.com/fatih/color v1.10.0 github.com/olekukonko/tablewriter v0.0.2-0.20190409134802-7e037d187b0c github.com/spf13/cobra v1.1.1 diff --git a/go.sum b/go.sum index 0488f32b..6d518986 100644 --- a/go.sum +++ b/go.sum @@ -80,14 +80,8 @@ github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+ github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cloudflare/cloudflare-go v0.10.2-0.20190916151808-a80f83b9add9/go.mod h1:1MxXX1Ux4x6mqPmjkUgTP1CdXIBXKX7T+Jk9Gxrmx+U= -github.com/coinbase/rosetta-sdk-go v0.6.2-0.20201201185830-8cd4faec860c h1:g9YLQ7Xlcbh4e7ehNghuUr0Tu7a0o+EKOV47q+3KjVg= -github.com/coinbase/rosetta-sdk-go v0.6.2-0.20201201185830-8cd4faec860c/go.mod h1:MvQfsL2KlJ5786OdDviRIJE3agui2YcvS1CaQPDl1Yo= -github.com/coinbase/rosetta-sdk-go v0.6.2-0.20201201193353-a11b3f3310c3 h1:wLrRp4gjVsX1/L3okNpXiaK5osCzeAl+h2XMocPYYxk= -github.com/coinbase/rosetta-sdk-go v0.6.2-0.20201201193353-a11b3f3310c3/go.mod h1:MvQfsL2KlJ5786OdDviRIJE3agui2YcvS1CaQPDl1Yo= -github.com/coinbase/rosetta-sdk-go v0.6.2-0.20201201193814-f4843ede4a3f h1:ku6sf059Al5duyD2a4/vV3p9yvGW1R/eMO3Cu67qPuM= -github.com/coinbase/rosetta-sdk-go v0.6.2-0.20201201193814-f4843ede4a3f/go.mod h1:MvQfsL2KlJ5786OdDviRIJE3agui2YcvS1CaQPDl1Yo= -github.com/coinbase/rosetta-sdk-go v0.6.2-0.20201201194354-4e3d0f143850 h1:2X4q0lyGzELPolyYFlSqE4bg+NCVyCT4+Mud2UxNI9Q= -github.com/coinbase/rosetta-sdk-go v0.6.2-0.20201201194354-4e3d0f143850/go.mod h1:MvQfsL2KlJ5786OdDviRIJE3agui2YcvS1CaQPDl1Yo= +github.com/coinbase/rosetta-sdk-go v0.6.2 h1:vgynGMbRkYv8S3/swrLGlww1iOKfurILLRCGJOrtLMk= +github.com/coinbase/rosetta-sdk-go v0.6.2/go.mod h1:MvQfsL2KlJ5786OdDviRIJE3agui2YcvS1CaQPDl1Yo= github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= From 5bcaa3c5d16dc49cb3ed2ee4a9c1d3da8a4daf3b Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Tue, 1 Dec 2020 17:56:18 -0600 Subject: [PATCH 213/376] update version --- cmd/root.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/root.go b/cmd/root.go index 58f440a1..ae8a2d39 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -257,6 +257,6 @@ var versionCmd = &cobra.Command{ Use: "version", Short: "Print rosetta-cli version", Run: func(cmd *cobra.Command, args []string) { - fmt.Println("v0.6.0") + fmt.Println("v0.6.1") }, } From 4fefdd75f4e2017d6349e8cd0fff550fb0d45424 Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Tue, 1 Dec 2020 17:59:52 -0600 Subject: [PATCH 214/376] shortened lines --- pkg/processor/balance_storage_handler.go | 26 ++++++++++++++++++++---- pkg/processor/balance_storage_helper.go | 10 +++++++-- pkg/tester/construction.go | 5 ++++- pkg/tester/data.go | 10 +++++++-- 4 files changed, 42 insertions(+), 9 deletions(-) diff --git a/pkg/processor/balance_storage_handler.go b/pkg/processor/balance_storage_handler.go index 6de2db22..84e0df18 100644 --- a/pkg/processor/balance_storage_handler.go +++ b/pkg/processor/balance_storage_handler.go @@ -113,13 +113,31 @@ func (h *BalanceStorageHandler) BlockRemoved( } // AccountsReconciled updates the total accounts reconciled by count. -func (h *BalanceStorageHandler) AccountsReconciled(ctx context.Context, dbTx database.Transaction, count int) error { - _, err := h.counterStorage.UpdateTransactional(ctx, dbTx, modules.ReconciledAccounts, big.NewInt(int64(count))) +func (h *BalanceStorageHandler) AccountsReconciled( + ctx context.Context, + dbTx database.Transaction, + count int, +) error { + _, err := h.counterStorage.UpdateTransactional( + ctx, + dbTx, + modules.ReconciledAccounts, + big.NewInt(int64(count)), + ) return err } // AccountsSeen updates the total accounts seen by count. -func (h *BalanceStorageHandler) AccountsSeen(ctx context.Context, dbTx database.Transaction, count int) error { - _, err := h.counterStorage.UpdateTransactional(ctx, dbTx, modules.SeenAccounts, big.NewInt(int64(count))) +func (h *BalanceStorageHandler) AccountsSeen( + ctx context.Context, + dbTx database.Transaction, + count int, +) error { + _, err := h.counterStorage.UpdateTransactional( + ctx, + dbTx, + modules.SeenAccounts, + big.NewInt(int64(count)), + ) return err } diff --git a/pkg/processor/balance_storage_helper.go b/pkg/processor/balance_storage_helper.go index b4d30195..f862093f 100644 --- a/pkg/processor/balance_storage_helper.go +++ b/pkg/processor/balance_storage_helper.go @@ -161,11 +161,17 @@ func (h *BalanceStorageHelper) BalanceExemptions() []*types.BalanceExemption { } // AccountsReconciled returns the total accounts reconciled by count. -func (h *BalanceStorageHelper) AccountsReconciled(ctx context.Context, dbTx database.Transaction) (*big.Int, error) { +func (h *BalanceStorageHelper) AccountsReconciled( + ctx context.Context, + dbTx database.Transaction, +) (*big.Int, error) { return h.counterStorage.GetTransactional(ctx, dbTx, modules.ReconciledAccounts) } // AccountsSeen returns the total accounts seen by count. -func (h *BalanceStorageHelper) AccountsSeen(ctx context.Context, dbTx database.Transaction) (*big.Int, error) { +func (h *BalanceStorageHelper) AccountsSeen( + ctx context.Context, + dbTx database.Transaction, +) (*big.Int, error) { return h.counterStorage.GetTransactional(ctx, dbTx, modules.SeenAccounts) } diff --git a/pkg/tester/construction.go b/pkg/tester/construction.go index ade038f9..11b40f17 100644 --- a/pkg/tester/construction.go +++ b/pkg/tester/construction.go @@ -91,7 +91,10 @@ func InitializeConstruction( opts = append(opts, database.WithoutCompression()) } if config.MemoryLimitDisabled { - opts = append(opts, database.WithCustomSettings(database.PerformanceBadgerOptions(dataPath))) + opts = append( + opts, + database.WithCustomSettings(database.PerformanceBadgerOptions(dataPath)), + ) } localStore, err := database.NewBadgerDatabase(ctx, dataPath, opts...) diff --git a/pkg/tester/data.go b/pkg/tester/data.go index 19b17d42..1ecabc9c 100644 --- a/pkg/tester/data.go +++ b/pkg/tester/data.go @@ -155,7 +155,10 @@ func InitializeData( opts = append(opts, database.WithoutCompression()) } if config.MemoryLimitDisabled { - opts = append(opts, database.WithCustomSettings(database.PerformanceBadgerOptions(dataPath))) + opts = append( + opts, + database.WithCustomSettings(database.PerformanceBadgerOptions(dataPath)), + ) } localStore, err := database.NewBadgerDatabase(ctx, dataPath, opts...) @@ -316,7 +319,10 @@ func InitializeData( statefulsyncer.WithPastBlockLimit(config.MaxReorgDepth), } if config.Data.PruningFrequency != nil { - statefulSyncerOptions = append(statefulSyncerOptions, statefulsyncer.WithPruneSleepTime(*config.Data.PruningFrequency)) + statefulSyncerOptions = append( + statefulSyncerOptions, + statefulsyncer.WithPruneSleepTime(*config.Data.PruningFrequency), + ) } syncer := statefulsyncer.New( From f8f51ecc76276d1245068a103076216ba153287f Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Tue, 1 Dec 2020 18:11:58 -0600 Subject: [PATCH 215/376] fix failing tests --- pkg/results/data_results_test.go | 87 ++++++++++++++++++++------------ 1 file changed, 56 insertions(+), 31 deletions(-) diff --git a/pkg/results/data_results_test.go b/pkg/results/data_results_test.go index ede6bc2e..d5b92c1b 100644 --- a/pkg/results/data_results_test.go +++ b/pkg/results/data_results_test.go @@ -26,6 +26,8 @@ import ( "github.com/coinbase/rosetta-sdk-go/asserter" "github.com/coinbase/rosetta-sdk-go/fetcher" + sdkMocks "github.com/coinbase/rosetta-sdk-go/mocks/storage/modules" + "github.com/coinbase/rosetta-sdk-go/parser" "github.com/coinbase/rosetta-sdk-go/storage/database" storageErrs "github.com/coinbase/rosetta-sdk-go/storage/errors" "github.com/coinbase/rosetta-sdk-go/storage/modules" @@ -33,8 +35,38 @@ import ( "github.com/coinbase/rosetta-sdk-go/types" "github.com/coinbase/rosetta-sdk-go/utils" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" ) +func baseAsserter() *asserter.Asserter { + a, _ := asserter.NewClientWithOptions( + &types.NetworkIdentifier{ + Blockchain: "bitcoin", + Network: "mainnet", + }, + &types.BlockIdentifier{ + Hash: "block 0", + Index: 0, + }, + []string{"Transfer"}, + []*types.OperationStatus{ + { + Status: "Success", + Successful: true, + }, + }, + []*types.Error{}, + nil, + ) + return a +} + +func exemptFunc() parser.ExemptOperation { + return func(op *types.Operation) bool { + return false + } +} + func TestComputeCheckDataResults(t *testing.T) { var tests = map[string]struct { cfg *configuration.Configuration @@ -411,37 +443,30 @@ func TestComputeCheckDataResults(t *testing.T) { var balanceStorage *modules.BalanceStorage if test.provideBalanceStorage { balanceStorage = modules.NewBalanceStorage(localStore) - - j := 0 - currency := &types.Currency{Symbol: "BLAH"} - block := &types.BlockIdentifier{Hash: "0", Index: 0} - for i := 0; i < test.totalAccounts; i++ { - dbTransaction := localStore.Transaction(ctx) - acct := &types.AccountIdentifier{ - Address: fmt.Sprintf("account %d", i), - } - assert.NoError(t, balanceStorage.SetBalance( - ctx, - dbTransaction, - acct, - &types.Amount{Value: "1", Currency: currency}, - block, - )) - assert.NoError(t, dbTransaction.Commit(ctx)) - - if j >= test.reconciledAccounts { - continue - } - - assert.NoError(t, balanceStorage.Reconciled( - ctx, - acct, - currency, - block, - )) - - j++ - } + mockHelper := &sdkMocks.BalanceStorageHelper{} + mockHelper.On("Asserter").Return(baseAsserter()) + mockHelper.On("ExemptFunc").Return(exemptFunc()) + mockHelper.On("BalanceExemptions").Return([]*types.BalanceExemption{}) + mockHelper.On( + "AccountsSeen", + mock.Anything, + mock.Anything, + mock.Anything, + ).Return( + big.NewInt(int64(test.totalAccounts)), + nil, + ) + mockHelper.On( + "AccountsReconciled", + mock.Anything, + mock.Anything, + mock.Anything, + ).Return( + big.NewInt(int64(test.reconciledAccounts)), + nil, + ) + mockHandler := &sdkMocks.BalanceStorageHandler{} + balanceStorage.Initialize(mockHelper, mockHandler) } t.Run(testName, func(t *testing.T) { From f7e768bcf8348d99a384734d345d7ef1d783f967 Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Wed, 2 Dec 2020 09:17:21 -0600 Subject: [PATCH 216/376] update rosetta-sdk-go --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index b19152f6..d3720907 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/coinbase/rosetta-cli go 1.13 require ( - github.com/coinbase/rosetta-sdk-go v0.6.2 + github.com/coinbase/rosetta-sdk-go v0.6.3 github.com/fatih/color v1.10.0 github.com/olekukonko/tablewriter v0.0.2-0.20190409134802-7e037d187b0c github.com/spf13/cobra v1.1.1 diff --git a/go.sum b/go.sum index 6d518986..7947da50 100644 --- a/go.sum +++ b/go.sum @@ -80,8 +80,8 @@ github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+ github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cloudflare/cloudflare-go v0.10.2-0.20190916151808-a80f83b9add9/go.mod h1:1MxXX1Ux4x6mqPmjkUgTP1CdXIBXKX7T+Jk9Gxrmx+U= -github.com/coinbase/rosetta-sdk-go v0.6.2 h1:vgynGMbRkYv8S3/swrLGlww1iOKfurILLRCGJOrtLMk= -github.com/coinbase/rosetta-sdk-go v0.6.2/go.mod h1:MvQfsL2KlJ5786OdDviRIJE3agui2YcvS1CaQPDl1Yo= +github.com/coinbase/rosetta-sdk-go v0.6.3 h1:PPj14tPJ7SFc8sY/hlwK8zddT7PKwWU2wicxyerDxlg= +github.com/coinbase/rosetta-sdk-go v0.6.3/go.mod h1:MvQfsL2KlJ5786OdDviRIJE3agui2YcvS1CaQPDl1Yo= github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= From 382f281182f850c9f36b696751abbe0e54efa3b3 Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Wed, 2 Dec 2020 09:18:54 -0600 Subject: [PATCH 217/376] update version --- cmd/root.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/root.go b/cmd/root.go index ae8a2d39..a1b6cd10 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -257,6 +257,6 @@ var versionCmd = &cobra.Command{ Use: "version", Short: "Print rosetta-cli version", Run: func(cmd *cobra.Command, args []string) { - fmt.Println("v0.6.1") + fmt.Println("v0.6.2") }, } From 1c8d86eacd6b67ef39bb94d2574ed1c93d1f00ee Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Wed, 2 Dec 2020 09:19:48 -0600 Subject: [PATCH 218/376] increase compilation target to go1.15.5 --- scripts/compile.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/compile.sh b/scripts/compile.sh index 151edf2e..c5b0d50c 100755 --- a/scripts/compile.sh +++ b/scripts/compile.sh @@ -16,7 +16,7 @@ VERSION=$1; -xgo -go go-1.15.2 --targets=darwin/*,windows/*,linux/* -out "bin/rosetta-cli-${VERSION}" .; +xgo -go go-1.15.5 --targets=darwin/*,windows/*,linux/* -out "bin/rosetta-cli-${VERSION}" .; # Rename some files mv "bin/rosetta-cli-${VERSION}-darwin-10.6-amd64" "bin/rosetta-cli-${VERSION}-darwin-amd64" From 5947d6eb7dec06b2c7b4afa667aa22ea4e256521 Mon Sep 17 00:00:00 2001 From: Omer Zach Date: Mon, 7 Dec 2020 14:14:45 -0500 Subject: [PATCH 219/376] Fix typo of "indicate" in README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 175f4599..e3f0509d 100644 --- a/README.md +++ b/README.md @@ -561,7 +561,7 @@ operations should be returned by the Rosetta Data API. Many of the packages use a `Helper/Handler` interface pattern to acquire required information or to send events to some client implementation. An example of this is in the `reconciler` package where a `Helper` is used to get -the account balance and the `Handler` is called to incidate whether the +the account balance and the `Handler` is called to indicate whether the reconciliation of an account was successful. ### Repo Structure From 528dfcc45bc3503e029eca6b4283936db01cb22b Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Mon, 7 Dec 2020 16:55:16 -0600 Subject: [PATCH 220/376] update rosetta-sdk-go@v0.6.4 --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index d3720907..3b85437c 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/coinbase/rosetta-cli go 1.13 require ( - github.com/coinbase/rosetta-sdk-go v0.6.3 + github.com/coinbase/rosetta-sdk-go v0.6.4 github.com/fatih/color v1.10.0 github.com/olekukonko/tablewriter v0.0.2-0.20190409134802-7e037d187b0c github.com/spf13/cobra v1.1.1 diff --git a/go.sum b/go.sum index 7947da50..4fc9123c 100644 --- a/go.sum +++ b/go.sum @@ -80,8 +80,8 @@ github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+ github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cloudflare/cloudflare-go v0.10.2-0.20190916151808-a80f83b9add9/go.mod h1:1MxXX1Ux4x6mqPmjkUgTP1CdXIBXKX7T+Jk9Gxrmx+U= -github.com/coinbase/rosetta-sdk-go v0.6.3 h1:PPj14tPJ7SFc8sY/hlwK8zddT7PKwWU2wicxyerDxlg= -github.com/coinbase/rosetta-sdk-go v0.6.3/go.mod h1:MvQfsL2KlJ5786OdDviRIJE3agui2YcvS1CaQPDl1Yo= +github.com/coinbase/rosetta-sdk-go v0.6.4 h1:+3z6otziRDdgEwy7IsyTkwgCNbbH4z5QMDnXdbdg9ZY= +github.com/coinbase/rosetta-sdk-go v0.6.4/go.mod h1:MvQfsL2KlJ5786OdDviRIJE3agui2YcvS1CaQPDl1Yo= github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= From 66a1ac3ca47f5b63ab78c2881ada07084e2fc9ce Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Mon, 7 Dec 2020 16:58:04 -0600 Subject: [PATCH 221/376] Add incomplete reconciler helper implementation --- pkg/processor/reconciler_helper.go | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/pkg/processor/reconciler_helper.go b/pkg/processor/reconciler_helper.go index 5254d4b3..60de2044 100644 --- a/pkg/processor/reconciler_helper.go +++ b/pkg/processor/reconciler_helper.go @@ -161,3 +161,18 @@ func (h *ReconcilerHelper) PruneBalances( index, ) } + +// ForceInactiveReconciliation overrides the default +// calculation to determine if an account should be +// reconciled inactively. +// +// TODO: complete this implementation (forcing when +// reaching tip if from_tip end condition is enabled) +func (h *ReconcilerHelper) ForceInactiveReconciliation( + ctx context.Context, + account *types.AccountIdentifier, + currency *types.Currency, + lastChecked *types.BlockIdentifier, +) bool { + return false +} From 9c9dcc38f04f411d7090133a4e0d7cf2749f7431 Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Mon, 7 Dec 2020 16:58:31 -0600 Subject: [PATCH 222/376] update versoin --- cmd/root.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/root.go b/cmd/root.go index a1b6cd10..2cd21059 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -257,6 +257,6 @@ var versionCmd = &cobra.Command{ Use: "version", Short: "Print rosetta-cli version", Run: func(cmd *cobra.Command, args []string) { - fmt.Println("v0.6.2") + fmt.Println("v0.6.3") }, } From 41e413fbfd586c7f08bf960d5457e00317835e88 Mon Sep 17 00:00:00 2001 From: Dan Sosedoff Date: Thu, 3 Dec 2020 12:40:19 -0600 Subject: [PATCH 223/376] Use configuration path provided by ROSETTA_CONFIGURATION_FILE --- cmd/root.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/cmd/root.go b/cmd/root.go index a1b6cd10..50a5edea 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -214,6 +214,9 @@ default values.`, func initConfig() { Context = context.Background() var err error + if len(configurationFile) == 0 { + configurationFile = os.Getenv("ROSETTA_CONFIGURATION_FILE") + } if len(configurationFile) == 0 { Config = configuration.DefaultConfiguration() } else { From 2ca2080d7f8aef3efab6491ac2232560010c5479 Mon Sep 17 00:00:00 2001 From: Dan Sosedoff Date: Mon, 7 Dec 2020 19:28:22 -0600 Subject: [PATCH 224/376] Address CR --- README.md | 11 +++++++++++ cmd/root.go | 11 ++++++++++- 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 175f4599..d2fde813 100644 --- a/README.md +++ b/README.md @@ -92,6 +92,17 @@ files for running tests against a Bitcoin Rosetta implementation ([config](https://github.com/coinbase/rosetta-bitcoin/tree/master/rosetta-cli-conf)) and an Ethereum Rosetta implementation ([config](https://github.com/coinbase/rosetta-ethereum/tree/master/rosetta-cli-conf)). +#### Using environment variables + +It's possible to set the configuration file path using an environment variable instead +of using a CLI flag. See an example below: + +```bash +ROSETTA_CONFIGURATION_FILE=/path/to/cli/config rosetta +``` + +CLI flags take precedence over environment variables. + #### Writing check:construction Tests The new Construction API testing framework (first released in `rosetta-cli@v0.5.0`) uses a new design pattern to allow for complex transaction construction orchestration. diff --git a/cmd/root.go b/cmd/root.go index 50a5edea..18a6c32b 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -31,6 +31,11 @@ import ( "github.com/spf13/cobra" ) +const ( + // configEnvKey is an env variable name that sets a config file location + configEnvKey = "ROSETTA_CONFIGURATION_FILE" +) + var ( rootCmd = &cobra.Command{ Use: "rosetta-cli", @@ -214,9 +219,13 @@ default values.`, func initConfig() { Context = context.Background() var err error + + // Use path provided by the environment variable if config path arg is not set. + // Default configuration will be used if the env var is not if len(configurationFile) == 0 { - configurationFile = os.Getenv("ROSETTA_CONFIGURATION_FILE") + configurationFile = os.Getenv(configEnvKey) } + if len(configurationFile) == 0 { Config = configuration.DefaultConfiguration() } else { From 12c2b8bc8644ae7f440548ec125f6750e297b32a Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Wed, 9 Dec 2020 09:22:04 -0600 Subject: [PATCH 225/376] Protect against invalid reconciliation configuration --- configuration/configuration.go | 4 +++ pkg/tester/data.go | 47 ++++++++++++++++++---------------- 2 files changed, 29 insertions(+), 22 deletions(-) diff --git a/configuration/configuration.go b/configuration/configuration.go index 1f6df0dd..0a983bcb 100644 --- a/configuration/configuration.go +++ b/configuration/configuration.go @@ -232,6 +232,10 @@ func assertDataConfiguration(config *DataConfiguration) error { return fmt.Errorf("start index %d cannot be negative", *config.StartIndex) } + if !config.ReconciliationDisabled && config.BalanceTrackingDisabled { + return errors.New("balance tracking must be enabled to perform reconciliation") + } + if config.EndConditions == nil { return nil } diff --git a/pkg/tester/data.go b/pkg/tester/data.go index 1ecabc9c..20a3fa83 100644 --- a/pkg/tester/data.go +++ b/pkg/tester/data.go @@ -180,28 +180,6 @@ func InitializeData( blockStorage := modules.NewBlockStorage(localStore) balanceStorage := modules.NewBalanceStorage(localStore) - // Bootstrap balances, if provided. We need to do before initializing - // the reconciler otherwise we won't reconcile bootstrapped accounts - // until rosetta-cli restart. - if len(config.Data.BootstrapBalances) > 0 { - _, err := blockStorage.GetHeadBlockIdentifier(ctx) - switch { - case err == storageErrs.ErrHeadBlockNotFound: - err = balanceStorage.BootstrapBalances( - ctx, - config.Data.BootstrapBalances, - genesisBlock, - ) - if err != nil { - log.Fatalf("%s: unable to bootstrap balances", err.Error()) - } - case err != nil: - log.Fatalf("%s: unable to get head block identifier", err.Error()) - default: - log.Println("Skipping balance bootstrapping because already started syncing") - } - } - logger := logger.NewLogger( dataPath, config.Data.LogBlocks, @@ -304,6 +282,31 @@ func InitializeData( balanceStorage.Initialize(balanceStorageHelper, balanceStorageHandler) blockWorkers = append(blockWorkers, balanceStorage) + + // Bootstrap balances, if provided. We need to do before initializing + // the reconciler otherwise we won't reconcile bootstrapped accounts + // until rosetta-cli restart. + // + // We need to do this after instantiating the balance storage handler + // because it is invoked within BootstrapBalances. + if len(config.Data.BootstrapBalances) > 0 { + _, err := blockStorage.GetHeadBlockIdentifier(ctx) + switch { + case err == storageErrs.ErrHeadBlockNotFound: + err = balanceStorage.BootstrapBalances( + ctx, + config.Data.BootstrapBalances, + genesisBlock, + ) + if err != nil { + log.Fatalf("%s: unable to bootstrap balances", err.Error()) + } + case err != nil: + log.Fatalf("%s: unable to get head block identifier", err.Error()) + default: + log.Println("Skipping balance bootstrapping because already started syncing") + } + } } if !config.Data.CoinTrackingDisabled { From de69f32a74c98e7216c9547d0f73fc7e13fbaa35 Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Wed, 9 Dec 2020 09:40:36 -0600 Subject: [PATCH 226/376] add todo --- pkg/results/data_results.go | 1 + 1 file changed, 1 insertion(+) diff --git a/pkg/results/data_results.go b/pkg/results/data_results.go index 1afdcdd1..7ab6fd93 100644 --- a/pkg/results/data_results.go +++ b/pkg/results/data_results.go @@ -250,6 +250,7 @@ func ComputeCheckDataStats( } if balances != nil { + // TODO: handle nil handler/nil helper error coverage, err := balances.EstimatedReconciliationCoverage(ctx) if err != nil { log.Printf("%s: cannot get reconcile coverage", err.Error()) From e73a5f0294a759b2edc35756bc74e9026cf67fb2 Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Wed, 9 Dec 2020 10:07:28 -0600 Subject: [PATCH 227/376] Handle ErrHelperHandlerMissing --- go.mod | 2 +- go.sum | 4 ++-- pkg/results/data_results.go | 13 ++++++++----- 3 files changed, 11 insertions(+), 8 deletions(-) diff --git a/go.mod b/go.mod index 3b85437c..f314b4cf 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/coinbase/rosetta-cli go 1.13 require ( - github.com/coinbase/rosetta-sdk-go v0.6.4 + github.com/coinbase/rosetta-sdk-go v0.6.5 github.com/fatih/color v1.10.0 github.com/olekukonko/tablewriter v0.0.2-0.20190409134802-7e037d187b0c github.com/spf13/cobra v1.1.1 diff --git a/go.sum b/go.sum index 4fc9123c..c21becea 100644 --- a/go.sum +++ b/go.sum @@ -80,8 +80,8 @@ github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+ github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cloudflare/cloudflare-go v0.10.2-0.20190916151808-a80f83b9add9/go.mod h1:1MxXX1Ux4x6mqPmjkUgTP1CdXIBXKX7T+Jk9Gxrmx+U= -github.com/coinbase/rosetta-sdk-go v0.6.4 h1:+3z6otziRDdgEwy7IsyTkwgCNbbH4z5QMDnXdbdg9ZY= -github.com/coinbase/rosetta-sdk-go v0.6.4/go.mod h1:MvQfsL2KlJ5786OdDviRIJE3agui2YcvS1CaQPDl1Yo= +github.com/coinbase/rosetta-sdk-go v0.6.5 h1:RytFDCPXS64vEYwIOsxsoQGlZZyP9RQvzyYikxymI4w= +github.com/coinbase/rosetta-sdk-go v0.6.5/go.mod h1:MvQfsL2KlJ5786OdDviRIJE3agui2YcvS1CaQPDl1Yo= github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= diff --git a/pkg/results/data_results.go b/pkg/results/data_results.go index 7ab6fd93..7c26bde3 100644 --- a/pkg/results/data_results.go +++ b/pkg/results/data_results.go @@ -250,14 +250,17 @@ func ComputeCheckDataStats( } if balances != nil { - // TODO: handle nil handler/nil helper error coverage, err := balances.EstimatedReconciliationCoverage(ctx) - if err != nil { - log.Printf("%s: cannot get reconcile coverage", err.Error()) + switch { + case err == nil: + stats.ReconciliationCoverage = coverage + case errors.Is(err, storageErrs.ErrHelperHandlerMissing): + // In this case, we use the default 0 value for the reconciliation + // coverage in stats. + case err != nil: + log.Printf("%s: cannot get reconciliation coverage", err.Error()) return nil } - - stats.ReconciliationCoverage = coverage } return stats From e94573bb6fe1e7fbe9e0081f0d97735753cc24a3 Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Wed, 9 Dec 2020 10:21:12 -0600 Subject: [PATCH 228/376] add serial and seen worker configs --- configuration/configuration.go | 21 +++++++++++++++++++++ configuration/configuration_test.go | 2 ++ configuration/types.go | 10 ++++++++++ 3 files changed, 33 insertions(+) diff --git a/configuration/configuration.go b/configuration/configuration.go index 0a983bcb..85e601e2 100644 --- a/configuration/configuration.go +++ b/configuration/configuration.go @@ -21,6 +21,7 @@ import ( "fmt" "log" "path" + "runtime" "github.com/coinbase/rosetta-sdk-go/asserter" "github.com/coinbase/rosetta-sdk-go/constructor/dsl" @@ -45,6 +46,7 @@ func DefaultDataConfiguration() *DataConfiguration { // EthereumNetwork, DefaultURL, DefaultTimeout, // DefaultConstructionConfiguration and DefaultDataConfiguration. func DefaultConfiguration() *Configuration { + numCPU := runtime.NumCPU() return &Configuration{ Network: EthereumNetwork, OnlineURL: DefaultURL, @@ -55,6 +57,8 @@ func DefaultConfiguration() *Configuration { TipDelay: DefaultTipDelay, MaxReorgDepth: DefaultMaxReorgDepth, Data: DefaultDataConfiguration(), + SeenBlockWorkers: numCPU, + SerialBlockWorkers: numCPU, } } @@ -153,6 +157,15 @@ func populateMissingFields(config *Configuration) *Configuration { config.MaxReorgDepth = DefaultMaxReorgDepth } + numCPU := runtime.NumCPU() + if config.SeenBlockWorkers == 0 { + config.SeenBlockWorkers = numCPU + } + + if config.SerialBlockWorkers == 0 { + config.SerialBlockWorkers = numCPU + } + config.Construction = populateConstructionMissingFields(config.Construction) config.Data = populateDataMissingFields(config.Data) @@ -292,6 +305,14 @@ func assertConfiguration(ctx context.Context, config *Configuration) error { return fmt.Errorf("%w: invalid network identifier", err) } + if config.SeenBlockWorkers <= 0 { + return errors.New("seen_block_workers must be > 0") + } + + if config.SerialBlockWorkers <= 0 { + return errors.New("serial_block_workers must be > 0") + } + if err := assertDataConfiguration(config.Data); err != nil { return fmt.Errorf("%w: invalid data configuration", err) } diff --git a/configuration/configuration_test.go b/configuration/configuration_test.go index 51703403..aa0486e7 100644 --- a/configuration/configuration_test.go +++ b/configuration/configuration_test.go @@ -68,6 +68,8 @@ var ( MaxSyncConcurrency: 12, TipDelay: 1231, MaxReorgDepth: 12, + SeenBlockWorkers: 300, + SerialBlockWorkers: 200, Construction: &ConstructionConfiguration{ OfflineURL: "https://ashdjaksdkjshdk", MaxOfflineConnections: 21, diff --git a/configuration/types.go b/configuration/types.go index 5a25f670..10c73a9d 100644 --- a/configuration/types.go +++ b/configuration/types.go @@ -396,6 +396,16 @@ type Configuration struct { // but can use 10s of GBs of RAM, even with pruning enabled. MemoryLimitDisabled bool `json:"memory_limit_disabled"` + // SeenBlockWorkers is the number of goroutines spawned to store + // seen blocks in storage before we attempt to sequence. If not populated, + // this value defaults to runtime.NumCPU(). + SeenBlockWorkers int `json:"seen_block_workers,omitempty"` + + // SerialBlockWorkers is the number of goroutines spawned to help + // with block sequencing (i.e. updating balances, updating coins, etc). + // If not populated, this value defaults to runtime.NumCPU(). + SerialBlockWorkers int `json:"serial_block_workers,omitempty"` + Construction *ConstructionConfiguration `json:"construction"` Data *DataConfiguration `json:"data"` } From f17d55204d5042f4956ff344d46e2469f0b2025b Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Wed, 9 Dec 2020 10:23:39 -0600 Subject: [PATCH 229/376] fix initializers --- pkg/tester/construction.go | 3 ++- pkg/tester/data.go | 6 ++++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/pkg/tester/construction.go b/pkg/tester/construction.go index 11b40f17..8b58c1e3 100644 --- a/pkg/tester/construction.go +++ b/pkg/tester/construction.go @@ -121,7 +121,7 @@ func InitializeConstruction( false, ) - blockStorage := modules.NewBlockStorage(localStore) + blockStorage := modules.NewBlockStorage(localStore, config.SerialBlockWorkers) keyStorage := modules.NewKeyStorage(localStore) coinStorageHelper := processor.NewCoinStorageHelper(blockStorage) coinStorage := modules.NewCoinStorage(localStore, coinStorageHelper, onlineFetcher.Asserter) @@ -266,6 +266,7 @@ func InitializeConstruction( statefulsyncer.WithCacheSize(syncer.DefaultCacheSize), statefulsyncer.WithMaxConcurrency(config.MaxSyncConcurrency), statefulsyncer.WithPastBlockLimit(config.MaxReorgDepth), + statefulsyncer.WithSeenConcurrency(int64(config.SeenBlockWorkers)), ) return &ConstructionTester{ diff --git a/pkg/tester/data.go b/pkg/tester/data.go index 20a3fa83..362e7847 100644 --- a/pkg/tester/data.go +++ b/pkg/tester/data.go @@ -177,7 +177,7 @@ func InitializeData( } counterStorage := modules.NewCounterStorage(localStore) - blockStorage := modules.NewBlockStorage(localStore) + blockStorage := modules.NewBlockStorage(localStore, config.SerialBlockWorkers) balanceStorage := modules.NewBalanceStorage(localStore) logger := logger.NewLogger( @@ -320,6 +320,7 @@ func InitializeData( statefulsyncer.WithCacheSize(syncer.DefaultCacheSize), statefulsyncer.WithMaxConcurrency(config.MaxSyncConcurrency), statefulsyncer.WithPastBlockLimit(config.MaxReorgDepth), + statefulsyncer.WithSeenConcurrency(int64(config.SeenBlockWorkers)), } if config.Data.PruningFrequency != nil { statefulSyncerOptions = append( @@ -978,7 +979,7 @@ func (t *DataTester) recursiveOpSearch( } counterStorage := modules.NewCounterStorage(localStore) - blockStorage := modules.NewBlockStorage(localStore) + blockStorage := modules.NewBlockStorage(localStore, t.config.SerialBlockWorkers) balanceStorage := modules.NewBalanceStorage(localStore) logger := logger.NewLogger( @@ -1055,6 +1056,7 @@ func (t *DataTester) recursiveOpSearch( statefulsyncer.WithCacheSize(syncer.DefaultCacheSize), statefulsyncer.WithMaxConcurrency(t.config.MaxSyncConcurrency), statefulsyncer.WithPastBlockLimit(t.config.MaxReorgDepth), + statefulsyncer.WithSeenConcurrency(int64(t.config.SeenBlockWorkers)), ) g, ctx := errgroup.WithContext(ctx) From f9dc03f7b684d69ab5cfb93542399f540d1535eb Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Wed, 9 Dec 2020 10:24:25 -0600 Subject: [PATCH 230/376] fix lint --- configuration/configuration.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configuration/configuration.go b/configuration/configuration.go index 85e601e2..0dd1cadf 100644 --- a/configuration/configuration.go +++ b/configuration/configuration.go @@ -240,7 +240,7 @@ func assertConstructionConfiguration(ctx context.Context, config *ConstructionCo return nil } -func assertDataConfiguration(config *DataConfiguration) error { +func assertDataConfiguration(config *DataConfiguration) error { // nolint:gocognit if config.StartIndex != nil && *config.StartIndex < 0 { return fmt.Errorf("start index %d cannot be negative", *config.StartIndex) } From 9bb073e8be533909fcf42bcd076c919bc0618c2f Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Wed, 9 Dec 2020 10:28:02 -0600 Subject: [PATCH 231/376] update version --- cmd/root.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/root.go b/cmd/root.go index c99f9c4b..dfe2249b 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -269,6 +269,6 @@ var versionCmd = &cobra.Command{ Use: "version", Short: "Print rosetta-cli version", Run: func(cmd *cobra.Command, args []string) { - fmt.Println("v0.6.3") + fmt.Println("v0.6.4") }, } From cd936bd492b99575626f69adb71ffe1461cde476 Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Wed, 9 Dec 2020 10:33:37 -0600 Subject: [PATCH 232/376] fix tests --- configuration/configuration.go | 3 --- configuration/configuration_test.go | 19 ++++++++++++++++++- 2 files changed, 18 insertions(+), 4 deletions(-) diff --git a/configuration/configuration.go b/configuration/configuration.go index 0dd1cadf..969aee7f 100644 --- a/configuration/configuration.go +++ b/configuration/configuration.go @@ -46,7 +46,6 @@ func DefaultDataConfiguration() *DataConfiguration { // EthereumNetwork, DefaultURL, DefaultTimeout, // DefaultConstructionConfiguration and DefaultDataConfiguration. func DefaultConfiguration() *Configuration { - numCPU := runtime.NumCPU() return &Configuration{ Network: EthereumNetwork, OnlineURL: DefaultURL, @@ -57,8 +56,6 @@ func DefaultConfiguration() *Configuration { TipDelay: DefaultTipDelay, MaxReorgDepth: DefaultMaxReorgDepth, Data: DefaultDataConfiguration(), - SeenBlockWorkers: numCPU, - SerialBlockWorkers: numCPU, } } diff --git a/configuration/configuration_test.go b/configuration/configuration_test.go index aa0486e7..bdc91253 100644 --- a/configuration/configuration_test.go +++ b/configuration/configuration_test.go @@ -18,6 +18,7 @@ import ( "context" "os/exec" "path" + "runtime" "testing" "github.com/coinbase/rosetta-sdk-go/constructor/job" @@ -158,7 +159,13 @@ func TestLoadConfiguration(t *testing.T) { }{ "nothing provided": { provided: &Configuration{}, - expected: DefaultConfiguration(), + expected: func() *Configuration { + cfg := DefaultConfiguration() + cfg.SeenBlockWorkers = runtime.NumCPU() + cfg.SerialBlockWorkers = runtime.NumCPU() + + return cfg + }(), }, "no overwrite": { provided: whackyConfig, @@ -173,6 +180,8 @@ func TestLoadConfiguration(t *testing.T) { }, expected: func() *Configuration { cfg := DefaultConfiguration() + cfg.SeenBlockWorkers = runtime.NumCPU() + cfg.SerialBlockWorkers = runtime.NumCPU() cfg.Construction = &ConstructionConfiguration{ OfflineURL: DefaultURL, MaxOfflineConnections: DefaultMaxOfflineConnections, @@ -195,6 +204,8 @@ func TestLoadConfiguration(t *testing.T) { }, expected: func() *Configuration { cfg := DefaultConfiguration() + cfg.SeenBlockWorkers = runtime.NumCPU() + cfg.SerialBlockWorkers = runtime.NumCPU() cfg.Construction = &ConstructionConfiguration{ OfflineURL: DefaultURL, MaxOfflineConnections: DefaultMaxOfflineConnections, @@ -223,6 +234,8 @@ func TestLoadConfiguration(t *testing.T) { }, expected: func() *Configuration { cfg := DefaultConfiguration() + cfg.SeenBlockWorkers = runtime.NumCPU() + cfg.SerialBlockWorkers = runtime.NumCPU() cfg.Construction = &ConstructionConfiguration{ OfflineURL: DefaultURL, MaxOfflineConnections: DefaultMaxOfflineConnections, @@ -314,6 +327,8 @@ func TestLoadConfiguration(t *testing.T) { }, expected: func() *Configuration { cfg := DefaultConfiguration() + cfg.SeenBlockWorkers = runtime.NumCPU() + cfg.SerialBlockWorkers = runtime.NumCPU() cfg.Data.EndConditions = &DataEndConditions{ ReconciliationCoverage: &ReconciliationCoverage{ Coverage: goodCoverage, @@ -371,6 +386,8 @@ func TestLoadConfiguration(t *testing.T) { provided: multipleEndConditions, expected: func() *Configuration { def := DefaultConfiguration() + def.SeenBlockWorkers = runtime.NumCPU() + def.SerialBlockWorkers = runtime.NumCPU() def.Data.EndConditions = multipleEndConditions.Data.EndConditions return def From 2db89f1bca5f4953932e9c4bafad7158a3c93abf Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Wed, 9 Dec 2020 11:13:45 -0600 Subject: [PATCH 233/376] Override at tip if sync status is synced --- pkg/tester/construction.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/pkg/tester/construction.go b/pkg/tester/construction.go index 8b58c1e3..b6867fd9 100644 --- a/pkg/tester/construction.go +++ b/pkg/tester/construction.go @@ -329,6 +329,12 @@ func (t *ConstructionTester) checkTip(ctx context.Context) (int64, error) { return status.CurrentBlockIdentifier.Index, nil } + // If the Rosetta implementation says it is at tip (regardless of the current + // block timestamp), we should start. + if status.SyncStatus != nil && status.SyncStatus.Synced != nil && *status.SyncStatus.Synced { + return status.CurrentBlockIdentifier.Index, nil + } + return -1, nil } From 6c38de3b76e782341d8b1d3c6d9ef2253b50bb7c Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Wed, 9 Dec 2020 11:57:50 -0600 Subject: [PATCH 234/376] force inactive reconciliation a tip --- pkg/processor/reconciler_helper.go | 30 ++++--- pkg/tester/data.go | 136 ++++++++++++++++++++--------- 2 files changed, 110 insertions(+), 56 deletions(-) diff --git a/pkg/processor/reconciler_helper.go b/pkg/processor/reconciler_helper.go index 60de2044..3fb163fc 100644 --- a/pkg/processor/reconciler_helper.go +++ b/pkg/processor/reconciler_helper.go @@ -37,9 +37,10 @@ type ReconcilerHelper struct { network *types.NetworkIdentifier fetcher *fetcher.Fetcher - database database.Database - blockStorage *modules.BlockStorage - balanceStorage *modules.BalanceStorage + database database.Database + blockStorage *modules.BlockStorage + balanceStorage *modules.BalanceStorage + forceInactiveReconciliation *bool } // NewReconcilerHelper returns a new ReconcilerHelper. @@ -50,14 +51,16 @@ func NewReconcilerHelper( database database.Database, blockStorage *modules.BlockStorage, balanceStorage *modules.BalanceStorage, + forceInactiveReconciliation *bool, ) *ReconcilerHelper { return &ReconcilerHelper{ - config: config, - network: network, - fetcher: fetcher, - database: database, - blockStorage: blockStorage, - balanceStorage: balanceStorage, + config: config, + network: network, + fetcher: fetcher, + database: database, + blockStorage: blockStorage, + balanceStorage: balanceStorage, + forceInactiveReconciliation: forceInactiveReconciliation, } } @@ -165,14 +168,15 @@ func (h *ReconcilerHelper) PruneBalances( // ForceInactiveReconciliation overrides the default // calculation to determine if an account should be // reconciled inactively. -// -// TODO: complete this implementation (forcing when -// reaching tip if from_tip end condition is enabled) func (h *ReconcilerHelper) ForceInactiveReconciliation( ctx context.Context, account *types.AccountIdentifier, currency *types.Currency, lastChecked *types.BlockIdentifier, ) bool { - return false + if h.forceInactiveReconciliation == nil { + return false + } + + return *h.forceInactiveReconciliation } diff --git a/pkg/tester/data.go b/pkg/tester/data.go index 362e7847..362286ad 100644 --- a/pkg/tester/data.go +++ b/pkg/tester/data.go @@ -72,22 +72,23 @@ var _ statefulsyncer.PruneHelper = (*DataTester)(nil) // DataTester coordinates the `check:data` test. type DataTester struct { - network *types.NetworkIdentifier - database database.Database - config *configuration.Configuration - syncer *statefulsyncer.StatefulSyncer - reconciler *reconciler.Reconciler - logger *logger.Logger - balanceStorage *modules.BalanceStorage - blockStorage *modules.BlockStorage - counterStorage *modules.CounterStorage - reconcilerHandler *processor.ReconcilerHandler - fetcher *fetcher.Fetcher - signalReceived *bool - genesisBlock *types.BlockIdentifier - cancel context.CancelFunc - historicalBalanceEnabled bool - parser *parser.Parser + network *types.NetworkIdentifier + database database.Database + config *configuration.Configuration + syncer *statefulsyncer.StatefulSyncer + reconciler *reconciler.Reconciler + logger *logger.Logger + balanceStorage *modules.BalanceStorage + blockStorage *modules.BlockStorage + counterStorage *modules.CounterStorage + reconcilerHandler *processor.ReconcilerHandler + fetcher *fetcher.Fetcher + signalReceived *bool + genesisBlock *types.BlockIdentifier + cancel context.CancelFunc + historicalBalanceEnabled bool + parser *parser.Parser + forceInactiveReconciliation *bool endCondition configuration.CheckDataEndCondition endConditionDetail string @@ -188,6 +189,7 @@ func InitializeData( config.Data.LogReconciliations, ) + var forceInactiveReconciliation bool reconcilerHelper := processor.NewReconcilerHelper( config, network, @@ -195,6 +197,7 @@ func InitializeData( localStore, blockStorage, balanceStorage, + &forceInactiveReconciliation, ) reconcilerHandler := processor.NewReconcilerHandler( @@ -342,22 +345,23 @@ func InitializeData( ) return &DataTester{ - network: network, - database: localStore, - config: config, - syncer: syncer, - cancel: cancel, - reconciler: r, - logger: logger, - balanceStorage: balanceStorage, - blockStorage: blockStorage, - counterStorage: counterStorage, - reconcilerHandler: reconcilerHandler, - fetcher: fetcher, - signalReceived: signalReceived, - genesisBlock: genesisBlock, - historicalBalanceEnabled: historicalBalanceEnabled, - parser: parser, + network: network, + database: localStore, + config: config, + syncer: syncer, + cancel: cancel, + reconciler: r, + logger: logger, + balanceStorage: balanceStorage, + blockStorage: blockStorage, + counterStorage: counterStorage, + reconcilerHandler: reconcilerHandler, + fetcher: fetcher, + signalReceived: signalReceived, + genesisBlock: genesisBlock, + historicalBalanceEnabled: historicalBalanceEnabled, + parser: parser, + forceInactiveReconciliation: &forceInactiveReconciliation, } } @@ -481,6 +485,44 @@ func (t *DataTester) ServeHTTP(w http.ResponseWriter, r *http.Request) { } } +// syncedToTip returns a boolean indicating if we are synced to tip and +// the last synced block. +func (t *DataTester) syncedToTip(ctx context.Context) (bool, int64, error) { + headBlock, err := t.blockStorage.GetBlock(ctx, nil) + if errors.Is(err, storageErrs.ErrHeadBlockNotFound) { + return false, -1, nil + } + if err != nil { + return false, -1, err + } + + blockIdentifier := headBlock.BlockIdentifier + + // If the last synced block is within TipDelay + // seconds from the current time, we are synced + // to tip. + if utils.AtTip(t.config.TipDelay, headBlock.Timestamp) { + return true, blockIdentifier.Index, nil + } + + status, fetchErr := t.fetcher.NetworkStatusRetry(ctx, t.network, nil) + if fetchErr != nil { + return false, -1, fmt.Errorf("%w: unable to fetch network status", fetchErr.Err) + } + + // If the Rosetta implementation says it is at tip (regardless of the current + // block timestamp) and our last synced block has the same index, + // we are synced to tip. + if status.SyncStatus != nil && + status.SyncStatus.Synced != nil && + *status.SyncStatus.Synced && + blockIdentifier.Index == status.CurrentBlockIdentifier.Index { + return true, blockIdentifier.Index, nil + } + + return false, blockIdentifier.Index, nil +} + // EndAtTipLoop runs a loop that evaluates end condition EndAtTip func (t *DataTester) EndAtTipLoop( ctx context.Context, @@ -494,7 +536,7 @@ func (t *DataTester) EndAtTipLoop( return case <-tc.C: - atTip, blockIdentifier, err := t.blockStorage.AtTip(ctx, t.config.TipDelay) + atTip, blockIndex, err := t.syncedToTip(ctx) if err != nil { log.Printf( "%s: unable to evaluate if syncer is at tip", @@ -507,7 +549,7 @@ func (t *DataTester) EndAtTipLoop( t.endCondition = configuration.TipEndCondition t.endConditionDetail = fmt.Sprintf( "Tip: %d", - blockIdentifier.Index, + blockIndex, ) t.cancel() return @@ -532,10 +574,7 @@ func (t *DataTester) EndReconciliationCoverage( // nolint:gocognit return case <-tc.C: - headBlock, err := t.blockStorage.GetBlock(ctx, nil) - if errors.Is(err, storageErrs.ErrHeadBlockNotFound) { - continue - } + atTip, blockIndex, err := t.syncedToTip(ctx) if err != nil { log.Printf( "%s: unable to evaluate syncer height or if at tip", @@ -544,24 +583,33 @@ func (t *DataTester) EndReconciliationCoverage( // nolint:gocognit continue } - blockIdentifier := headBlock.BlockIdentifier - atTip := utils.AtTip(t.config.TipDelay, headBlock.Timestamp) - // Check if we are at tip and set tip height if fromTip is true. if reconciliationCoverage.Tip || reconciliationCoverage.FromTip { // If we fall behind tip, we must reset the firstTipIndex. + var disableForceReconciliation bool if !atTip { + disableForceReconciliation = true firstTipIndex = int64(-1) continue } + // forceInactiveReconciliation should NEVER be nil + // by this point but we check just to be sure. + if t.forceInactiveReconciliation != nil { + *t.forceInactiveReconciliation = !disableForceReconciliation + + if !disableForceReconciliation { + log.Println("enabling forced inactive reconciliation") + } + } + // Once at tip, we want to consider // coverage. It is not feasible that we could // get high reconciliation coverage at the tip // block, so we take the range from when first // at tip to the current block. if firstTipIndex < 0 { - firstTipIndex = blockIdentifier.Index + firstTipIndex = blockIndex } } @@ -571,7 +619,7 @@ func (t *DataTester) EndReconciliationCoverage( // nolint:gocognit // Check if at required minimum index if reconciliationCoverage.Index != nil { - if *reconciliationCoverage.Index < blockIdentifier.Index { + if *reconciliationCoverage.Index < blockIndex { continue } @@ -990,6 +1038,7 @@ func (t *DataTester) recursiveOpSearch( false, ) + t.forceInactiveReconciliation = types.Bool(false) reconcilerHelper := processor.NewReconcilerHelper( t.config, t.network, @@ -997,6 +1046,7 @@ func (t *DataTester) recursiveOpSearch( localStore, blockStorage, balanceStorage, + t.forceInactiveReconciliation, ) reconcilerHandler := processor.NewReconcilerHandler( From 21fd7c6398ab55f8e430c8ca600c12a581a757d6 Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Wed, 9 Dec 2020 12:00:13 -0600 Subject: [PATCH 235/376] nits --- pkg/tester/data.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/pkg/tester/data.go b/pkg/tester/data.go index 362286ad..51e69140 100644 --- a/pkg/tester/data.go +++ b/pkg/tester/data.go @@ -485,9 +485,9 @@ func (t *DataTester) ServeHTTP(w http.ResponseWriter, r *http.Request) { } } -// syncedToTip returns a boolean indicating if we are synced to tip and +// syncedStatus returns a boolean indicating if we are synced to tip and // the last synced block. -func (t *DataTester) syncedToTip(ctx context.Context) (bool, int64, error) { +func (t *DataTester) syncedStatus(ctx context.Context) (bool, int64, error) { headBlock, err := t.blockStorage.GetBlock(ctx, nil) if errors.Is(err, storageErrs.ErrHeadBlockNotFound) { return false, -1, nil @@ -536,7 +536,7 @@ func (t *DataTester) EndAtTipLoop( return case <-tc.C: - atTip, blockIndex, err := t.syncedToTip(ctx) + atTip, blockIndex, err := t.syncedStatus(ctx) if err != nil { log.Printf( "%s: unable to evaluate if syncer is at tip", @@ -574,7 +574,7 @@ func (t *DataTester) EndReconciliationCoverage( // nolint:gocognit return case <-tc.C: - atTip, blockIndex, err := t.syncedToTip(ctx) + atTip, blockIndex, err := t.syncedStatus(ctx) if err != nil { log.Printf( "%s: unable to evaluate syncer height or if at tip", From ac446c9050d2e17a28141374180400900f775bab Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Fri, 11 Dec 2020 08:58:22 -0600 Subject: [PATCH 236/376] cleanup at tip printing for reconciliation --- pkg/tester/data.go | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/pkg/tester/data.go b/pkg/tester/data.go index 51e69140..67e2ba6b 100644 --- a/pkg/tester/data.go +++ b/pkg/tester/data.go @@ -597,10 +597,6 @@ func (t *DataTester) EndReconciliationCoverage( // nolint:gocognit // by this point but we check just to be sure. if t.forceInactiveReconciliation != nil { *t.forceInactiveReconciliation = !disableForceReconciliation - - if !disableForceReconciliation { - log.Println("enabling forced inactive reconciliation") - } } // Once at tip, we want to consider @@ -676,6 +672,13 @@ func (t *DataTester) EndReconciliationCoverage( // nolint:gocognit t.cancel() return } + + color.Cyan(fmt.Sprintf( + "[END CONDITIONS] Waiting for reconciliation coverage after block %d (%f%%) to surpass required coverage (%f%%)", + firstTipIndex, + reconciliationCoverage.Coverage*utils.OneHundred, + coverage*utils.OneHundred, + )) } } } From 5dd106d99823990a2b24040b80814470a700cf98 Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Fri, 11 Dec 2020 09:01:15 -0600 Subject: [PATCH 237/376] print out seen accounts --- pkg/logger/logger.go | 3 ++- pkg/results/data_results.go | 11 +++++++++++ 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/pkg/logger/logger.go b/pkg/logger/logger.go index eda0378f..e5ef440d 100644 --- a/pkg/logger/logger.go +++ b/pkg/logger/logger.go @@ -97,11 +97,12 @@ func (l *Logger) LogDataStatus(ctx context.Context, status *results.CheckDataSta } statsMessage := fmt.Sprintf( - "[STATS] Blocks: %d (Orphaned: %d) Transactions: %d Operations: %d Reconciliations: %d (Inactive: %d, Exempt: %d, Skipped: %d, Coverage: %f%%)", // nolint:lll + "[STATS] Blocks: %d (Orphaned: %d) Transactions: %d Operations: %d Accounts: %d Reconciliations: %d (Inactive: %d, Exempt: %d, Skipped: %d, Coverage: %f%%)", // nolint:lll status.Stats.Blocks, status.Stats.Orphans, status.Stats.Transactions, status.Stats.Operations, + status.Stats.Accounts, status.Stats.ActiveReconciliations+status.Stats.InactiveReconciliations, status.Stats.InactiveReconciliations, status.Stats.ExemptReconciliations, diff --git a/pkg/results/data_results.go b/pkg/results/data_results.go index 7c26bde3..4d1ef51f 100644 --- a/pkg/results/data_results.go +++ b/pkg/results/data_results.go @@ -101,6 +101,7 @@ type CheckDataStats struct { Orphans int64 `json:"orphans"` Transactions int64 `json:"transactions"` Operations int64 `json:"operations"` + Accounts int64 `json:"accounts"` ActiveReconciliations int64 `json:"active_reconciliations"` InactiveReconciliations int64 `json:"inactive_reconciliations"` ExemptReconciliations int64 `json:"exempt_reconciliations"` @@ -127,6 +128,9 @@ func (c *CheckDataStats) Print() { table.Append( []string{"Operations", "# of operations processed", strconv.FormatInt(c.Operations, 10)}, ) + table.Append( + []string{"Accounts", "# of accounts seen", strconv.FormatInt(c.Accounts, 10)}, + ) table.Append( []string{ "Active Reconciliations", @@ -207,6 +211,12 @@ func ComputeCheckDataStats( return nil } + accounts, err := counters.Get(ctx, modules.SeenAccounts) + if err != nil { + log.Printf("%s: cannot get accounts counter", err.Error()) + return nil + } + activeReconciliations, err := counters.Get(ctx, modules.ActiveReconciliationCounter) if err != nil { log.Printf("%s: cannot get active reconciliations counter", err.Error()) @@ -242,6 +252,7 @@ func ComputeCheckDataStats( Orphans: orphans.Int64(), Transactions: txs.Int64(), Operations: ops.Int64(), + Accounts: accounts.Int64(), ActiveReconciliations: activeReconciliations.Int64(), InactiveReconciliations: inactiveReconciliations.Int64(), ExemptReconciliations: exemptReconciliations.Int64(), From dc27654e5e51fa9c74eeae8bad0b010fc3cf4144 Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Fri, 11 Dec 2020 09:02:47 -0600 Subject: [PATCH 238/376] coverage nit --- pkg/tester/data.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/tester/data.go b/pkg/tester/data.go index 67e2ba6b..13e3ff43 100644 --- a/pkg/tester/data.go +++ b/pkg/tester/data.go @@ -674,10 +674,10 @@ func (t *DataTester) EndReconciliationCoverage( // nolint:gocognit } color.Cyan(fmt.Sprintf( - "[END CONDITIONS] Waiting for reconciliation coverage after block %d (%f%%) to surpass required coverage (%f%%)", + "[END CONDITIONS] Waiting for reconciliation coverage after block %d (%f%%) to surpass requirement (%f%%)", firstTipIndex, - reconciliationCoverage.Coverage*utils.OneHundred, coverage*utils.OneHundred, + reconciliationCoverage.Coverage*utils.OneHundred, )) } } From 6fd37f4023232f5263799afcbbcecc541960cbdb Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Fri, 11 Dec 2020 15:26:06 -0600 Subject: [PATCH 239/376] update rosetta-sdk-go --- go.mod | 2 +- go.sum | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index f314b4cf..73934f77 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/coinbase/rosetta-cli go 1.13 require ( - github.com/coinbase/rosetta-sdk-go v0.6.5 + github.com/coinbase/rosetta-sdk-go v0.6.6 github.com/fatih/color v1.10.0 github.com/olekukonko/tablewriter v0.0.2-0.20190409134802-7e037d187b0c github.com/spf13/cobra v1.1.1 diff --git a/go.sum b/go.sum index c21becea..09f550f8 100644 --- a/go.sum +++ b/go.sum @@ -82,6 +82,8 @@ github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDk github.com/cloudflare/cloudflare-go v0.10.2-0.20190916151808-a80f83b9add9/go.mod h1:1MxXX1Ux4x6mqPmjkUgTP1CdXIBXKX7T+Jk9Gxrmx+U= github.com/coinbase/rosetta-sdk-go v0.6.5 h1:RytFDCPXS64vEYwIOsxsoQGlZZyP9RQvzyYikxymI4w= github.com/coinbase/rosetta-sdk-go v0.6.5/go.mod h1:MvQfsL2KlJ5786OdDviRIJE3agui2YcvS1CaQPDl1Yo= +github.com/coinbase/rosetta-sdk-go v0.6.6 h1:DUuC1LyK1tm/9XzPIuWiC8qOqjaFHvVWH0oA4X4kXJc= +github.com/coinbase/rosetta-sdk-go v0.6.6/go.mod h1:Spu6Ha6TB3T7TVrzgWzzNvebPuueFbKHyKAfuWQHOE0= github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= @@ -369,6 +371,7 @@ github.com/tidwall/gjson v1.6.1 h1:LRbvNuNuvAiISWg6gxLEFuCe72UKy5hDqhxW/8183ws= github.com/tidwall/gjson v1.6.1/go.mod h1:BaHyNc5bjzYkPqgLq7mdVzeiRtULKULXLgZFKsxEHI0= github.com/tidwall/gjson v1.6.3 h1:aHoiiem0dr7GHkW001T1SMTJ7X5PvyekH5WX0whWGnI= github.com/tidwall/gjson v1.6.3/go.mod h1:BaHyNc5bjzYkPqgLq7mdVzeiRtULKULXLgZFKsxEHI0= +github.com/tidwall/gjson v1.6.4/go.mod h1:BaHyNc5bjzYkPqgLq7mdVzeiRtULKULXLgZFKsxEHI0= github.com/tidwall/match v1.0.1 h1:PnKP62LPNxHKTwvHHZZzdOAOCtsJTjo6dZLCwpKm5xc= github.com/tidwall/match v1.0.1/go.mod h1:LujAq0jyVjBy028G1WhWfIzbpQfMO8bBZ6Tyb0+pL9E= github.com/tidwall/pretty v1.0.2 h1:Z7S3cePv9Jwm1KwS0513MRaoUe3S01WPbLNV40pwWZU= From 797d8d49a9568a8527083d6155b11d546e985bed Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Fri, 11 Dec 2020 15:26:48 -0600 Subject: [PATCH 240/376] update rosetta-cli version --- cmd/root.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/root.go b/cmd/root.go index dfe2249b..866d1acc 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -269,6 +269,6 @@ var versionCmd = &cobra.Command{ Use: "version", Short: "Print rosetta-cli version", Run: func(cmd *cobra.Command, args []string) { - fmt.Println("v0.6.4") + fmt.Println("v0.6.5") }, } From 99edf7726dcec0cbbb72925e1527757926b49394 Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Fri, 11 Dec 2020 15:27:10 -0600 Subject: [PATCH 241/376] go mod tidy --- go.sum | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/go.sum b/go.sum index 09f550f8..37c4736a 100644 --- a/go.sum +++ b/go.sum @@ -80,8 +80,6 @@ github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+ github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cloudflare/cloudflare-go v0.10.2-0.20190916151808-a80f83b9add9/go.mod h1:1MxXX1Ux4x6mqPmjkUgTP1CdXIBXKX7T+Jk9Gxrmx+U= -github.com/coinbase/rosetta-sdk-go v0.6.5 h1:RytFDCPXS64vEYwIOsxsoQGlZZyP9RQvzyYikxymI4w= -github.com/coinbase/rosetta-sdk-go v0.6.5/go.mod h1:MvQfsL2KlJ5786OdDviRIJE3agui2YcvS1CaQPDl1Yo= github.com/coinbase/rosetta-sdk-go v0.6.6 h1:DUuC1LyK1tm/9XzPIuWiC8qOqjaFHvVWH0oA4X4kXJc= github.com/coinbase/rosetta-sdk-go v0.6.6/go.mod h1:Spu6Ha6TB3T7TVrzgWzzNvebPuueFbKHyKAfuWQHOE0= github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= @@ -369,8 +367,7 @@ github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69 github.com/syndtr/goleveldb v1.0.1-0.20200815110645-5c35d600f0ca/go.mod h1:u2MKkTVTVJWe5D1rCvame8WqhBd88EuIwODJZ1VHCPM= github.com/tidwall/gjson v1.6.1 h1:LRbvNuNuvAiISWg6gxLEFuCe72UKy5hDqhxW/8183ws= github.com/tidwall/gjson v1.6.1/go.mod h1:BaHyNc5bjzYkPqgLq7mdVzeiRtULKULXLgZFKsxEHI0= -github.com/tidwall/gjson v1.6.3 h1:aHoiiem0dr7GHkW001T1SMTJ7X5PvyekH5WX0whWGnI= -github.com/tidwall/gjson v1.6.3/go.mod h1:BaHyNc5bjzYkPqgLq7mdVzeiRtULKULXLgZFKsxEHI0= +github.com/tidwall/gjson v1.6.4 h1:JKsCsJqRVFz8eYCsQ5E/ANRbK6CanAtA9IUvGsXklyo= github.com/tidwall/gjson v1.6.4/go.mod h1:BaHyNc5bjzYkPqgLq7mdVzeiRtULKULXLgZFKsxEHI0= github.com/tidwall/match v1.0.1 h1:PnKP62LPNxHKTwvHHZZzdOAOCtsJTjo6dZLCwpKm5xc= github.com/tidwall/match v1.0.1/go.mod h1:LujAq0jyVjBy028G1WhWfIzbpQfMO8bBZ6Tyb0+pL9E= From 8d394855a7a3499b4906d921d6a5caf468827469 Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Fri, 11 Dec 2020 15:53:37 -0600 Subject: [PATCH 242/376] update CoordinatorHelper --- pkg/processor/coordinator_helper.go | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/pkg/processor/coordinator_helper.go b/pkg/processor/coordinator_helper.go index c017e8c9..6d204fc3 100644 --- a/pkg/processor/coordinator_helper.go +++ b/pkg/processor/coordinator_helper.go @@ -53,6 +53,8 @@ const ( argUnsignedTransaction = "unsigned_transaction" argTransactionIdentifier = "transaction_identifier" argNetworkTransaction = "network_transaction" + + kvPrefix = "coordinator_kv" ) var _ coordinator.Helper = (*CoordinatorHelper)(nil) @@ -512,3 +514,30 @@ func (c *CoordinatorHelper) HeadBlockExists(ctx context.Context) bool { return headBlock != nil } + +func kvKey(key string) []byte { + return []byte(fmt.Sprintf("%s/%s", kvPrefix, key)) +} + +// SetBlob transactionally persists +// a key and value. +func (c *CoordinatorHelper) SetBlob( + ctx context.Context, + dbTx database.Transaction, + key string, + value []byte, +) error { + // We defensively don't claim the value slice + // in our buffer pool. + return dbTx.Set(ctx, kvKey(key), value, false) +} + +// GetBlob transactionally retrieves +// a key and value. +func (c *CoordinatorHelper) GetBlob( + ctx context.Context, + dbTx database.Transaction, + key string, +) (bool, []byte, error) { + return dbTx.Get(ctx, kvKey(key)) +} From 8cb8bd0f0737fa13940a09d73aa488896bc190d2 Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Fri, 11 Dec 2020 15:55:54 -0600 Subject: [PATCH 243/376] add ForceRetry to config --- configuration/types.go | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/configuration/types.go b/configuration/types.go index 10c73a9d..da4d1b2e 100644 --- a/configuration/types.go +++ b/configuration/types.go @@ -84,6 +84,10 @@ type ConstructionConfiguration struct { // fetcher will open. MaxOfflineConnections int `json:"max_offline_connections"` + // ForceRetry overrides the default retry handling to retry + // on all non-200 responses. + ForceRetry bool `json:"force_retry,omitempty"` + // StaleDepth is the number of blocks to wait before attempting // to rebroadcast after not finding a transaction on-chain. StaleDepth int64 `json:"stale_depth"` @@ -363,6 +367,10 @@ type Configuration struct { // fetcher will open. MaxOnlineConnections int `json:"max_online_connections"` + // ForceRetry overrides the default retry handling to retry + // on all non-200 responses. + ForceRetry bool `json:"force_retry,omitempty"` + // MaxSyncConcurrency is the maximum sync concurrency to use while syncing blocks. // Sync concurrency is managed automatically by the `syncer` package. MaxSyncConcurrency int64 `json:"max_sync_concurrency"` From 5da94a1b4d251716bbf28c469b515f18f5660540 Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Fri, 11 Dec 2020 16:00:58 -0600 Subject: [PATCH 244/376] Add ForceRetry --- cmd/check_construction.go | 15 +++++++++++---- cmd/check_data.go | 15 +++++++++++---- cmd/view_balance.go | 14 +++++++++++--- cmd/view_block.go | 14 +++++++++++--- cmd/view_networks.go | 14 +++++++++++--- pkg/tester/construction.go | 14 +++++++++++--- 6 files changed, 66 insertions(+), 20 deletions(-) diff --git a/cmd/check_construction.go b/cmd/check_construction.go index c26d13b7..dc32d3ab 100644 --- a/cmd/check_construction.go +++ b/cmd/check_construction.go @@ -66,12 +66,19 @@ func runCheckConstructionCmd(cmd *cobra.Command, args []string) error { ensureDataDirectoryExists() ctx, cancel := context.WithCancel(Context) - fetcher := fetcher.New( - Config.OnlineURL, + fetcherOpts := []fetcher.Option{ fetcher.WithMaxConnections(Config.MaxOnlineConnections), - fetcher.WithRetryElapsedTime(time.Duration(Config.RetryElapsedTime)*time.Second), - fetcher.WithTimeout(time.Duration(Config.HTTPTimeout)*time.Second), + fetcher.WithRetryElapsedTime(time.Duration(Config.RetryElapsedTime) * time.Second), + fetcher.WithTimeout(time.Duration(Config.HTTPTimeout) * time.Second), fetcher.WithMaxRetries(Config.MaxRetries), + } + if Config.ForceRetry { + fetcherOpts = append(fetcherOpts, fetcher.WithForceRetry()) + } + + fetcher := fetcher.New( + Config.OnlineURL, + fetcherOpts..., ) _, _, fetchErr := fetcher.InitializeAsserter(ctx, Config.Network) diff --git a/cmd/check_data.go b/cmd/check_data.go index 2277ffdd..75b56d0c 100644 --- a/cmd/check_data.go +++ b/cmd/check_data.go @@ -74,12 +74,19 @@ func runCheckDataCmd(cmd *cobra.Command, args []string) error { ensureDataDirectoryExists() ctx, cancel := context.WithCancel(Context) - fetcher := fetcher.New( - Config.OnlineURL, + fetcherOpts := []fetcher.Option{ fetcher.WithMaxConnections(Config.MaxOnlineConnections), - fetcher.WithRetryElapsedTime(time.Duration(Config.RetryElapsedTime)*time.Second), - fetcher.WithTimeout(time.Duration(Config.HTTPTimeout)*time.Second), + fetcher.WithRetryElapsedTime(time.Duration(Config.RetryElapsedTime) * time.Second), + fetcher.WithTimeout(time.Duration(Config.HTTPTimeout) * time.Second), fetcher.WithMaxRetries(Config.MaxRetries), + } + if Config.ForceRetry { + fetcherOpts = append(fetcherOpts, fetcher.WithForceRetry()) + } + + fetcher := fetcher.New( + Config.OnlineURL, + fetcherOpts..., ) _, _, fetchErr := fetcher.InitializeAsserter(ctx, Config.Network) diff --git a/cmd/view_balance.go b/cmd/view_balance.go index 61ca8cb0..32ca3da1 100644 --- a/cmd/view_balance.go +++ b/cmd/view_balance.go @@ -56,11 +56,19 @@ func runViewBalanceCmd(cmd *cobra.Command, args []string) error { } // Create a new fetcher + fetcherOpts := []fetcher.Option{ + fetcher.WithMaxConnections(Config.MaxOnlineConnections), + fetcher.WithRetryElapsedTime(time.Duration(Config.RetryElapsedTime) * time.Second), + fetcher.WithTimeout(time.Duration(Config.HTTPTimeout) * time.Second), + fetcher.WithMaxRetries(Config.MaxRetries), + } + if Config.ForceRetry { + fetcherOpts = append(fetcherOpts, fetcher.WithForceRetry()) + } + newFetcher := fetcher.New( Config.OnlineURL, - fetcher.WithRetryElapsedTime(time.Duration(Config.RetryElapsedTime)*time.Second), - fetcher.WithTimeout(time.Duration(Config.HTTPTimeout)*time.Second), - fetcher.WithMaxRetries(Config.MaxRetries), + fetcherOpts..., ) // Initialize the fetcher's asserter diff --git a/cmd/view_block.go b/cmd/view_block.go index 51262c9f..17be0f26 100644 --- a/cmd/view_block.go +++ b/cmd/view_block.go @@ -73,11 +73,19 @@ func runViewBlockCmd(cmd *cobra.Command, args []string) error { } // Create a new fetcher + fetcherOpts := []fetcher.Option{ + fetcher.WithMaxConnections(Config.MaxOnlineConnections), + fetcher.WithRetryElapsedTime(time.Duration(Config.RetryElapsedTime) * time.Second), + fetcher.WithTimeout(time.Duration(Config.HTTPTimeout) * time.Second), + fetcher.WithMaxRetries(Config.MaxRetries), + } + if Config.ForceRetry { + fetcherOpts = append(fetcherOpts, fetcher.WithForceRetry()) + } + newFetcher := fetcher.New( Config.OnlineURL, - fetcher.WithRetryElapsedTime(time.Duration(Config.RetryElapsedTime)*time.Second), - fetcher.WithTimeout(time.Duration(Config.HTTPTimeout)*time.Second), - fetcher.WithMaxRetries(Config.MaxRetries), + fetcherOpts..., ) // Initialize the fetcher's asserter diff --git a/cmd/view_networks.go b/cmd/view_networks.go index 6dab0d82..eda06a50 100644 --- a/cmd/view_networks.go +++ b/cmd/view_networks.go @@ -41,11 +41,19 @@ not formatted correctly.`, ) func runViewNetworksCmd(cmd *cobra.Command, args []string) error { + fetcherOpts := []fetcher.Option{ + fetcher.WithMaxConnections(Config.MaxOnlineConnections), + fetcher.WithRetryElapsedTime(time.Duration(Config.RetryElapsedTime) * time.Second), + fetcher.WithTimeout(time.Duration(Config.HTTPTimeout) * time.Second), + fetcher.WithMaxRetries(Config.MaxRetries), + } + if Config.ForceRetry { + fetcherOpts = append(fetcherOpts, fetcher.WithForceRetry()) + } + f := fetcher.New( Config.OnlineURL, - fetcher.WithRetryElapsedTime(time.Duration(Config.RetryElapsedTime)*time.Second), - fetcher.WithTimeout(time.Duration(Config.HTTPTimeout)*time.Second), - fetcher.WithMaxRetries(Config.MaxRetries), + fetcherOpts..., ) // Attempt to fetch network list diff --git a/pkg/tester/construction.go b/pkg/tester/construction.go index b6867fd9..509a9f18 100644 --- a/pkg/tester/construction.go +++ b/pkg/tester/construction.go @@ -162,12 +162,20 @@ func InitializeConstruction( blockStorage, onlineFetcher, ) - offlineFetcher := fetcher.New( - config.Construction.OfflineURL, + + fetcherOpts := []fetcher.Option{ fetcher.WithMaxConnections(config.Construction.MaxOfflineConnections), fetcher.WithAsserter(onlineFetcher.Asserter), - fetcher.WithTimeout(time.Duration(config.HTTPTimeout)*time.Second), + fetcher.WithTimeout(time.Duration(config.HTTPTimeout) * time.Second), fetcher.WithMaxRetries(config.MaxRetries), + } + if config.Construction.ForceRetry { + fetcherOpts = append(fetcherOpts, fetcher.WithForceRetry()) + } + + offlineFetcher := fetcher.New( + config.Construction.OfflineURL, + fetcherOpts..., ) // Import prefunded account and save to database From f1418270a1ffcf469dd860a3be59e33400cf153c Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Mon, 14 Dec 2020 13:08:28 -0800 Subject: [PATCH 245/376] update rosetta-sdk-go --- cmd/root.go | 2 +- go.mod | 2 +- go.sum | 14 ++++++++------ 3 files changed, 10 insertions(+), 8 deletions(-) diff --git a/cmd/root.go b/cmd/root.go index 866d1acc..018358ab 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -269,6 +269,6 @@ var versionCmd = &cobra.Command{ Use: "version", Short: "Print rosetta-cli version", Run: func(cmd *cobra.Command, args []string) { - fmt.Println("v0.6.5") + fmt.Println("v0.6.6") }, } diff --git a/go.mod b/go.mod index 73934f77..2df8bef6 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/coinbase/rosetta-cli go 1.13 require ( - github.com/coinbase/rosetta-sdk-go v0.6.6 + github.com/coinbase/rosetta-sdk-go v0.6.7 github.com/fatih/color v1.10.0 github.com/olekukonko/tablewriter v0.0.2-0.20190409134802-7e037d187b0c github.com/spf13/cobra v1.1.1 diff --git a/go.sum b/go.sum index 37c4736a..7a1efc7a 100644 --- a/go.sum +++ b/go.sum @@ -80,8 +80,8 @@ github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+ github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cloudflare/cloudflare-go v0.10.2-0.20190916151808-a80f83b9add9/go.mod h1:1MxXX1Ux4x6mqPmjkUgTP1CdXIBXKX7T+Jk9Gxrmx+U= -github.com/coinbase/rosetta-sdk-go v0.6.6 h1:DUuC1LyK1tm/9XzPIuWiC8qOqjaFHvVWH0oA4X4kXJc= -github.com/coinbase/rosetta-sdk-go v0.6.6/go.mod h1:Spu6Ha6TB3T7TVrzgWzzNvebPuueFbKHyKAfuWQHOE0= +github.com/coinbase/rosetta-sdk-go v0.6.7 h1:v2fzv1cOVWB4A2QjIEcZpuh/CFwfLmX+Jd6qP3AZHN0= +github.com/coinbase/rosetta-sdk-go v0.6.7/go.mod h1:DbkDDurZUlfMqFrWfSU3gLsLCaGDyw3Cqz/f1JWjo5U= github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= @@ -119,8 +119,8 @@ github.com/dvyukov/go-fuzz v0.0.0-20200318091601-be3528f3a813/go.mod h1:11Gm+ccJ github.com/edsrzf/mmap-go v0.0.0-20160512033002-935e0e8a636c/go.mod h1:YO35OhQPt3KJa3ryjFM5Bs14WD66h8eGKpfaBNrHW5M= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= -github.com/ethereum/go-ethereum v1.9.24 h1:6AK+ORt3EMDO+FTjzXy/AQwHMbu52J2nYHIjyQX9azQ= -github.com/ethereum/go-ethereum v1.9.24/go.mod h1:JIfVb6esrqALTExdz9hRYvrP0xBDf6wCncIu1hNwHpM= +github.com/ethereum/go-ethereum v1.9.25 h1:mMiw/zOOtCLdGLWfcekua0qPrJTe7FVIiHJ4IKNTfR0= +github.com/ethereum/go-ethereum v1.9.25/go.mod h1:vMkFiYLHI4tgPw4k2j4MHKoovchFE8plZ0M9VMk4/oM= github.com/fatih/color v1.3.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/fatih/color v1.10.0 h1:s36xzo75JdqLaaWoiEHk767eHiwo0598uUxyfiPkDsg= @@ -159,8 +159,8 @@ github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QD github.com/golang/protobuf v1.4.2 h1:+Z5KGCizgyZCbGh1KZqA0fcLLkwbsjIzS4aV2v7wJX0= github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= -github.com/golang/snappy v0.0.2-0.20200707131729-196ae77b8a26 h1:lMm2hD9Fy0ynom5+85/pbdkiYcBqM1JWmhpAXLmy0fw= -github.com/golang/snappy v0.0.2-0.20200707131729-196ae77b8a26/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/golang/snappy v0.0.3-0.20201103224600-674baa8c7fc3 h1:ur2rms48b3Ep1dxh7aUV2FZEQ8jEVO2F6ILKx8ofkAg= +github.com/golang/snappy v0.0.3-0.20201103224600-674baa8c7fc3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= @@ -218,6 +218,7 @@ github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NH github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/influxdata/influxdb v1.2.3-0.20180221223340-01288bdb0883/go.mod h1:qZna6X/4elxqT3yI9iZYdZrWWdeFOOprn86kgg4+IzY= github.com/jackpal/go-nat-pmp v1.0.2-0.20160603034137-1fa385a6f458/go.mod h1:QPH045xvCAeXUZOxsnwmrtiCoxIr9eob+4orBN1SBKc= +github.com/jedisct1/go-minisign v0.0.0-20190909160543-45766022959e/go.mod h1:G1CVv03EnqU1wYL2dFwXxW2An0az9JTl/ZsqXQeBlkU= github.com/jessevdk/go-flags v0.0.0-20141203071132-1679536dcc89/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= @@ -402,6 +403,7 @@ golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACk golang.org/x/crypto v0.0.0-20190426145343-a29dc8fdc734/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190909091759-094676da4a83/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550 h1:ObdrDkeb4kJdCP557AjRjq69pTHfNouLtWZG7j9rPN8= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200115085410-6d4e4cb37c7d/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= From 23e6daf53528a2695b05e6a7f584d4f5e31c1023 Mon Sep 17 00:00:00 2001 From: Madhur Shrimal Date: Thu, 7 Jan 2021 14:28:37 -0800 Subject: [PATCH 246/376] Add troubleshoot section (#210) * add-troubleshoot-section * update to 10000 * fix check option --- .gitignore | 3 +++ README.md | 9 +++++++++ 2 files changed, 12 insertions(+) diff --git a/.gitignore b/.gitignore index 218fd2fa..0891b757 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,5 @@ rosetta-cli bin/ + +# Intellij setting +.idea/ diff --git a/README.md b/README.md index 5f8fcdc9..2d957dee 100644 --- a/README.md +++ b/README.md @@ -585,6 +585,15 @@ pkg tester // test orchestrators ``` +## Troubleshoot +* While running `check:data` or `check:construction` option if you get the following error: + + ```dial tcp 127.0.0.1:8080: socket: too many open files: unable to sync to 1902533: unable to sync to 1902533``` + + Please run `ulimit -n 10000` to increase the max concurrent opened file limit + + _Note: MacOS users, if you face `ulimit: setrlimit failed: invalid argument` error while setting `ulimit`, please run `sudo launchctl limit maxfiles 10000 200000` before setting the `ulimit`_ + ## License This project is available open source under the terms of the [Apache 2.0 License](https://opensource.org/licenses/Apache-2.0). From f5f4d16073b96d65c47666713a31daeffc160bea Mon Sep 17 00:00:00 2001 From: swapna gupta Date: Tue, 12 Jan 2021 14:08:50 -0500 Subject: [PATCH 247/376] Replace at-tip checks with utils functions from the latest sdk (#212) * replace at-tip checks with utils functions from latest sdk * check for nil blockidentifier * lint * update version --- cmd/root.go | 2 +- go.mod | 2 +- go.sum | 22 ++++++-------- pkg/processor/broadcast_storage_helper.go | 8 +++-- pkg/tester/construction.go | 23 +++++++------- pkg/tester/data.go | 37 +++++++---------------- 6 files changed, 40 insertions(+), 54 deletions(-) diff --git a/cmd/root.go b/cmd/root.go index 018358ab..2b6bf31b 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -269,6 +269,6 @@ var versionCmd = &cobra.Command{ Use: "version", Short: "Print rosetta-cli version", Run: func(cmd *cobra.Command, args []string) { - fmt.Println("v0.6.6") + fmt.Println("v0.6.7") }, } diff --git a/go.mod b/go.mod index 2df8bef6..ee7d452f 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/coinbase/rosetta-cli go 1.13 require ( - github.com/coinbase/rosetta-sdk-go v0.6.7 + github.com/coinbase/rosetta-sdk-go v0.6.8 github.com/fatih/color v1.10.0 github.com/olekukonko/tablewriter v0.0.2-0.20190409134802-7e037d187b0c github.com/spf13/cobra v1.1.1 diff --git a/go.sum b/go.sum index 7a1efc7a..e23cc8fb 100644 --- a/go.sum +++ b/go.sum @@ -80,8 +80,8 @@ github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+ github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cloudflare/cloudflare-go v0.10.2-0.20190916151808-a80f83b9add9/go.mod h1:1MxXX1Ux4x6mqPmjkUgTP1CdXIBXKX7T+Jk9Gxrmx+U= -github.com/coinbase/rosetta-sdk-go v0.6.7 h1:v2fzv1cOVWB4A2QjIEcZpuh/CFwfLmX+Jd6qP3AZHN0= -github.com/coinbase/rosetta-sdk-go v0.6.7/go.mod h1:DbkDDurZUlfMqFrWfSU3gLsLCaGDyw3Cqz/f1JWjo5U= +github.com/coinbase/rosetta-sdk-go v0.6.8 h1:DCWvfpzN9HkKux0eBZwN+9b7vstP4ij77zpyHtYy8dk= +github.com/coinbase/rosetta-sdk-go v0.6.8/go.mod h1:jczGuxueJURzr7J8/OeZM3FRbezjw8lWIMxXOBLONOM= github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= @@ -366,23 +366,21 @@ github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= github.com/syndtr/goleveldb v1.0.1-0.20200815110645-5c35d600f0ca/go.mod h1:u2MKkTVTVJWe5D1rCvame8WqhBd88EuIwODJZ1VHCPM= -github.com/tidwall/gjson v1.6.1 h1:LRbvNuNuvAiISWg6gxLEFuCe72UKy5hDqhxW/8183ws= -github.com/tidwall/gjson v1.6.1/go.mod h1:BaHyNc5bjzYkPqgLq7mdVzeiRtULKULXLgZFKsxEHI0= -github.com/tidwall/gjson v1.6.4 h1:JKsCsJqRVFz8eYCsQ5E/ANRbK6CanAtA9IUvGsXklyo= -github.com/tidwall/gjson v1.6.4/go.mod h1:BaHyNc5bjzYkPqgLq7mdVzeiRtULKULXLgZFKsxEHI0= -github.com/tidwall/match v1.0.1 h1:PnKP62LPNxHKTwvHHZZzdOAOCtsJTjo6dZLCwpKm5xc= -github.com/tidwall/match v1.0.1/go.mod h1:LujAq0jyVjBy028G1WhWfIzbpQfMO8bBZ6Tyb0+pL9E= +github.com/tidwall/gjson v1.6.7 h1:Mb1M9HZCRWEcXQ8ieJo7auYyyiSux6w9XN3AdTpxJrE= +github.com/tidwall/gjson v1.6.7/go.mod h1:zeFuBCIqD4sN/gmqBzZ4j7Jd6UcA2Fc56x7QFsv+8fI= +github.com/tidwall/match v1.0.3 h1:FQUVvBImDutD8wJLN6c5eMzWtjgONK9MwIBCOrUJKeE= +github.com/tidwall/match v1.0.3/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM= github.com/tidwall/pretty v1.0.2 h1:Z7S3cePv9Jwm1KwS0513MRaoUe3S01WPbLNV40pwWZU= github.com/tidwall/pretty v1.0.2/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= -github.com/tidwall/sjson v1.1.2 h1:NC5okI+tQ8OG/oyzchvwXXxRxCV/FVdhODbPKkQ25jQ= -github.com/tidwall/sjson v1.1.2/go.mod h1:SEzaDwxiPzKzNfUEO4HbYF/m4UCSJDsGgNqsS1LvdoY= +github.com/tidwall/sjson v1.1.4 h1:bTSsPLdAYF5QNLSwYsKfBKKTnlGbIuhqL3CpRsjzGhg= +github.com/tidwall/sjson v1.1.4/go.mod h1:wXpKXu8CtDjKAZ+3DrKY5ROCorDFahq8l0tey/Lx1fg= github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/tyler-smith/go-bip39 v1.0.1-0.20181017060643-dbb3b84ba2ef/go.mod h1:sJ5fKU0s6JVwZjjcUEX2zFOnvq0ASQ2K9Zr6cf67kNs= github.com/tyler-smith/go-bip39 v1.0.2/go.mod h1:sJ5fKU0s6JVwZjjcUEX2zFOnvq0ASQ2K9Zr6cf67kNs= github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= -github.com/vmihailenco/msgpack/v5 v5.1.0 h1:+od5YbEXxW95SPlW6beocmt8nOtlh83zqat5Ip9Hwdc= -github.com/vmihailenco/msgpack/v5 v5.1.0/go.mod h1:C5gboKD0TJPqWDTVTtrQNfRbiBwHZGo8UTqP/9/XvLI= +github.com/vmihailenco/msgpack/v5 v5.1.4 h1:6K44/cU6dMNGkVTGGuu7ef2NdSRFMhAFGGLfE3cqtHM= +github.com/vmihailenco/msgpack/v5 v5.1.4/go.mod h1:C5gboKD0TJPqWDTVTtrQNfRbiBwHZGo8UTqP/9/XvLI= github.com/vmihailenco/tagparser v0.1.2 h1:gnjoVuB/kljJ5wICEEOpx98oXMWPLj22G67Vbd1qPqc= github.com/vmihailenco/tagparser v0.1.2/go.mod h1:OeAg3pn3UbLjkWt+rN9oFYB6u/cQgqMEUPoW2WPyhdI= github.com/wsddn/go-ecdh v0.0.0-20161211032359-48726bab9208/go.mod h1:IotVbo4F+mw0EzQ08zFqg7pK3FebNXpaMsRy2RT+Ees= diff --git a/pkg/processor/broadcast_storage_helper.go b/pkg/processor/broadcast_storage_helper.go index 91eecb2f..5e881852 100644 --- a/pkg/processor/broadcast_storage_helper.go +++ b/pkg/processor/broadcast_storage_helper.go @@ -17,6 +17,7 @@ package processor import ( "context" "fmt" + "github.com/coinbase/rosetta-sdk-go/utils" "github.com/coinbase/rosetta-sdk-go/fetcher" "github.com/coinbase/rosetta-sdk-go/storage/database" @@ -29,16 +30,19 @@ var _ modules.BroadcastStorageHelper = (*BroadcastStorageHelper)(nil) // BroadcastStorageHelper implements the storage.Helper // interface. type BroadcastStorageHelper struct { + network *types.NetworkIdentifier blockStorage *modules.BlockStorage fetcher *fetcher.Fetcher } // NewBroadcastStorageHelper returns a new BroadcastStorageHelper. func NewBroadcastStorageHelper( + network *types.NetworkIdentifier, blockStorage *modules.BlockStorage, fetcher *fetcher.Fetcher, ) *BroadcastStorageHelper { return &BroadcastStorageHelper{ + network: network, blockStorage: blockStorage, fetcher: fetcher, } @@ -49,9 +53,9 @@ func (h *BroadcastStorageHelper) AtTip( ctx context.Context, tipDelay int64, ) (bool, error) { - atTip, _, err := h.blockStorage.AtTip(ctx, tipDelay) + atTip, _, err := utils.CheckStorageTip(ctx, h.network, tipDelay, h.fetcher, h.blockStorage) if err != nil { - return false, fmt.Errorf("%w: unable to determine if at tip", err) + return false, err } return atTip, nil diff --git a/pkg/tester/construction.go b/pkg/tester/construction.go index 509a9f18..9f997d7a 100644 --- a/pkg/tester/construction.go +++ b/pkg/tester/construction.go @@ -159,6 +159,7 @@ func InitializeConstruction( parser := parser.New(onlineFetcher.Asserter, nil, networkOptions.Allow.BalanceExemptions) broadcastHelper := processor.NewBroadcastStorageHelper( + network, blockStorage, onlineFetcher, ) @@ -327,20 +328,18 @@ func (t *ConstructionTester) StartPeriodicLogger( } func (t *ConstructionTester) checkTip(ctx context.Context) (int64, error) { - status, fetchErr := t.onlineFetcher.NetworkStatusRetry(ctx, t.network, nil) - if fetchErr != nil { - return -1, fmt.Errorf("%w: unable to fetch network status", fetchErr.Err) - } - - // If a block has yet to be synced, start syncing from tip. - if utils.AtTip(t.config.TipDelay, status.CurrentBlockTimestamp) { - return status.CurrentBlockIdentifier.Index, nil + atTip, blockIdentifier, err := utils.CheckNetworkTip( + ctx, + t.network, + t.config.TipDelay, + t.onlineFetcher, + ) + if err != nil { + return -1, err } - // If the Rosetta implementation says it is at tip (regardless of the current - // block timestamp), we should start. - if status.SyncStatus != nil && status.SyncStatus.Synced != nil && *status.SyncStatus.Synced { - return status.CurrentBlockIdentifier.Index, nil + if atTip { + return blockIdentifier.Index, nil } return -1, nil diff --git a/pkg/tester/data.go b/pkg/tester/data.go index 13e3ff43..336664cc 100644 --- a/pkg/tester/data.go +++ b/pkg/tester/data.go @@ -488,39 +488,24 @@ func (t *DataTester) ServeHTTP(w http.ResponseWriter, r *http.Request) { // syncedStatus returns a boolean indicating if we are synced to tip and // the last synced block. func (t *DataTester) syncedStatus(ctx context.Context) (bool, int64, error) { - headBlock, err := t.blockStorage.GetBlock(ctx, nil) - if errors.Is(err, storageErrs.ErrHeadBlockNotFound) { - return false, -1, nil - } + atTip, blockIdentifier, err := utils.CheckStorageTip( + ctx, + t.network, + t.config.TipDelay, + t.fetcher, + t.blockStorage, + ) if err != nil { return false, -1, err } - blockIdentifier := headBlock.BlockIdentifier - - // If the last synced block is within TipDelay - // seconds from the current time, we are synced - // to tip. - if utils.AtTip(t.config.TipDelay, headBlock.Timestamp) { - return true, blockIdentifier.Index, nil - } - - status, fetchErr := t.fetcher.NetworkStatusRetry(ctx, t.network, nil) - if fetchErr != nil { - return false, -1, fmt.Errorf("%w: unable to fetch network status", fetchErr.Err) - } + var blockIndex int64 = -1 - // If the Rosetta implementation says it is at tip (regardless of the current - // block timestamp) and our last synced block has the same index, - // we are synced to tip. - if status.SyncStatus != nil && - status.SyncStatus.Synced != nil && - *status.SyncStatus.Synced && - blockIdentifier.Index == status.CurrentBlockIdentifier.Index { - return true, blockIdentifier.Index, nil + if blockIdentifier != nil { + blockIndex = blockIdentifier.Index } - return false, blockIdentifier.Index, nil + return atTip, blockIndex, nil } // EndAtTipLoop runs a loop that evaluates end condition EndAtTip From b2b75c41b1972ce481358ee1faf9eb9392c45f11 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 14 Jan 2021 05:19:49 +0000 Subject: [PATCH 248/376] Bump github.com/stretchr/testify from 1.6.1 to 1.7.0 Bumps [github.com/stretchr/testify](https://github.com/stretchr/testify) from 1.6.1 to 1.7.0. - [Release notes](https://github.com/stretchr/testify/releases) - [Commits](https://github.com/stretchr/testify/compare/v1.6.1...v1.7.0) Signed-off-by: dependabot[bot] --- go.mod | 2 +- go.sum | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index ee7d452f..73c5a30b 100644 --- a/go.mod +++ b/go.mod @@ -7,6 +7,6 @@ require ( github.com/fatih/color v1.10.0 github.com/olekukonko/tablewriter v0.0.2-0.20190409134802-7e037d187b0c github.com/spf13/cobra v1.1.1 - github.com/stretchr/testify v1.6.1 + github.com/stretchr/testify v1.7.0 golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208 ) diff --git a/go.sum b/go.sum index e23cc8fb..585aabe9 100644 --- a/go.sum +++ b/go.sum @@ -364,6 +364,8 @@ github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81P github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= github.com/syndtr/goleveldb v1.0.1-0.20200815110645-5c35d600f0ca/go.mod h1:u2MKkTVTVJWe5D1rCvame8WqhBd88EuIwODJZ1VHCPM= github.com/tidwall/gjson v1.6.7 h1:Mb1M9HZCRWEcXQ8ieJo7auYyyiSux6w9XN3AdTpxJrE= From 21c61b0092ad161b57c553edd128c076846058c4 Mon Sep 17 00:00:00 2001 From: Hao Wen Date: Tue, 9 Mar 2021 13:01:19 -0800 Subject: [PATCH 249/376] Sort array fields for asserter configuration --- Makefile | 4 +- cmd/utils_asserter_configuration.go | 16 +++- cmd/utils_asserter_configuration_test.go | 111 +++++++++++++++++++++++ 3 files changed, 128 insertions(+), 3 deletions(-) create mode 100644 cmd/utils_asserter_configuration_test.go diff --git a/Makefile b/Makefile index 22429225..644b0718 100644 --- a/Makefile +++ b/Makefile @@ -12,8 +12,8 @@ ADDLICENCE_SCRIPT=${ADDLICENSE_CMD} -c "Coinbase, Inc." -l "apache" -v GOLINES_CMD=go run github.com/segmentio/golines GOVERALLS_CMD=go run github.com/mattn/goveralls COVERAGE_TEST_DIRECTORIES=./configuration/... ./pkg/results/... \ - ./pkg/logger/... -TEST_SCRIPT=go test -v ./pkg/... ./configuration/... + ./pkg/logger/... ./cmd +TEST_SCRIPT=go test -v ./pkg/... ./configuration/... ./cmd COVERAGE_TEST_SCRIPT=go test -v ${COVERAGE_TEST_DIRECTORIES} deps: diff --git a/cmd/utils_asserter_configuration.go b/cmd/utils_asserter_configuration.go index 6fde4099..ff70365f 100644 --- a/cmd/utils_asserter_configuration.go +++ b/cmd/utils_asserter_configuration.go @@ -16,8 +16,10 @@ package cmd import ( "fmt" + "sort" "time" + "github.com/coinbase/rosetta-sdk-go/asserter" "github.com/coinbase/rosetta-sdk-go/fetcher" "github.com/coinbase/rosetta-sdk-go/utils" "github.com/fatih/color" @@ -30,7 +32,7 @@ var ( Short: "Generate a static configuration file for the Asserter", Long: `In production deployments, it is useful to initialize the response Asserter (https://github.com/coinbase/rosetta-sdk-go/tree/master/asserter) using -a static configuration instead of intializing a configuration dynamically +a static configuration instead of initializing a configuration dynamically from the node. This allows a client to error on new types/statuses that may have been added in an update instead of silently erroring. @@ -61,6 +63,8 @@ func runCreateConfigurationCmd(cmd *cobra.Command, args []string) error { return fmt.Errorf("%w: unable to generate spec", err) } + sortArrayFieldsOnConfiguration(configuration) + if err := utils.SerializeAndWrite(args[0], configuration); err != nil { return fmt.Errorf("%w: unable to serialize asserter configuration", err) } @@ -68,3 +72,13 @@ func runCreateConfigurationCmd(cmd *cobra.Command, args []string) error { color.Green("Configuration file saved!") return nil } + +func sortArrayFieldsOnConfiguration(configuration *asserter.Configuration) { + sort.Strings(configuration.AllowedOperationTypes) + sort.Slice(configuration.AllowedOperationStatuses, func(i, j int) bool { + return configuration.AllowedOperationStatuses[i].Status < configuration.AllowedOperationStatuses[j].Status + }) + sort.Slice(configuration.AllowedErrors, func(i, j int) bool { + return configuration.AllowedErrors[i].Code < configuration.AllowedErrors[j].Code + }) +} \ No newline at end of file diff --git a/cmd/utils_asserter_configuration_test.go b/cmd/utils_asserter_configuration_test.go new file mode 100644 index 00000000..ce6f3cab --- /dev/null +++ b/cmd/utils_asserter_configuration_test.go @@ -0,0 +1,111 @@ +// Copyright 2020 Coinbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package cmd + +import "github.com/coinbase/rosetta-sdk-go/types" + +import ( + "testing" + + "github.com/stretchr/testify/assert" + + "github.com/coinbase/rosetta-sdk-go/asserter" +) + +var ( + basicNetwork = &types.NetworkIdentifier{ + Blockchain: "blockchain", + Network: "network", + } + + basicBlock = &types.BlockIdentifier{ + Index: 10, + Hash: "block 10", + } + + allowedOperationTypes = []string{"OUTPUT", "INPUT", "TRANSFER"} + + allowedOperationStatuses = []*types.OperationStatus{ + { + Status: "SUCCESS", + Successful: true, + }, + { + Status: "SKIPPED", + Successful: true, + }, + } + + allowedErrors = []*types.Error{ + { + Code: 4, + Message: "Block not found", + Retriable: false, + }, + { + Code: 0, + Message: "Endpoint not implemented", + Retriable: false, + }, + { + Code: 3, + Message: "Bitcoind error", + Retriable: false, + }, + } + + timestampStartIndex = int64(6) +) + +func TestSortArrayFields(t *testing.T) { + var clientConfiguration = &asserter.Configuration{ + NetworkIdentifier: basicNetwork, + GenesisBlockIdentifier: basicBlock, + AllowedOperationTypes: allowedOperationTypes, + AllowedOperationStatuses: allowedOperationStatuses, + AllowedErrors: allowedErrors, + AllowedTimestampStartIndex: timestampStartIndex, + } + var assert = assert.New(t) + sortArrayFieldsOnConfiguration(clientConfiguration) + assert.Equal([]string{"INPUT", "OUTPUT", "TRANSFER"}, clientConfiguration.AllowedOperationTypes) + assert.Equal([]*types.OperationStatus{ + { + Status: "SKIPPED", + Successful: true, + }, + { + Status: "SUCCESS", + Successful: true, + }, + }, clientConfiguration.AllowedOperationStatuses) + assert.Equal([]*types.Error{ + { + Code: 0, + Message: "Endpoint not implemented", + Retriable: false, + }, + { + Code: 3, + Message: "Bitcoind error", + Retriable: false, + }, + { + Code: 4, + Message: "Block not found", + Retriable: false, + }, + }, clientConfiguration.AllowedErrors) +} From 1374ba5daa02d7662eb555e15fdf0fd084807b7b Mon Sep 17 00:00:00 2001 From: Raghav Sharma Date: Tue, 9 Mar 2021 15:53:11 -0800 Subject: [PATCH 250/376] Fix PROTO-1835 --- cmd/view_block.go | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/cmd/view_block.go b/cmd/view_block.go index 17be0f26..3de96074 100644 --- a/cmd/view_block.go +++ b/cmd/view_block.go @@ -15,6 +15,7 @@ package cmd import ( + "errors" "fmt" "strconv" "time" @@ -66,7 +67,7 @@ func printChanges(balanceChanges []*parser.BalanceChange) error { return nil } -func runViewBlockCmd(cmd *cobra.Command, args []string) error { +func runViewBlockCmd(_ *cobra.Command, args []string) error { index, err := strconv.ParseInt(args[0], 10, 64) if err != nil { return fmt.Errorf("%w: unable to parse index %s", err, args[0]) @@ -122,6 +123,10 @@ func runViewBlockCmd(cmd *cobra.Command, args []string) error { if fetchErr != nil { return fmt.Errorf("%w: unable to fetch block", fetchErr.Err) } + // It's valid for a block to be omitted without triggering an error + if block == nil { + return errors.New("block not found, it might be omitted") + } fmt.Printf("\n") if !OnlyChanges { From 48191debbf0016f241c03ad858e831c131810138 Mon Sep 17 00:00:00 2001 From: Raghav Sharma Date: Thu, 11 Mar 2021 15:05:52 -0800 Subject: [PATCH 251/376] Draft of PROTO-1751 --- cmd/check_data.go | 18 +++- cmd/root.go | 16 ++++ cmd/validate_asserter_config.go | 162 ++++++++++++++++++++++++++++++++ 3 files changed, 195 insertions(+), 1 deletion(-) create mode 100644 cmd/validate_asserter_config.go diff --git a/cmd/check_data.go b/cmd/check_data.go index 75b56d0c..ecbbd131 100644 --- a/cmd/check_data.go +++ b/cmd/check_data.go @@ -70,7 +70,7 @@ of what one of these files looks like.`, } ) -func runCheckDataCmd(cmd *cobra.Command, args []string) error { +func runCheckDataCmd(_ *cobra.Command, _ []string) error { ensureDataDirectoryExists() ctx, cancel := context.WithCancel(Context) @@ -115,6 +115,22 @@ func runCheckDataCmd(cmd *cobra.Command, args []string) error { ) } + if asserterConfigurationPath != "" { + if err := validateAsserterConfiguration( + ctx, fetcher, Config.Network, asserterConfigurationPath, + ); err != nil { + cancel() + return results.ExitData( + Config, + nil, + nil, + fmt.Errorf("%w: network options mismatch asserter configuration", err), + "", + "", + ) + } + } + dataTester := tester.InitializeData( ctx, Config, diff --git a/cmd/root.go b/cmd/root.go index 2b6bf31b..5b8b13aa 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -71,6 +71,10 @@ var ( // OnlyChanges is a boolean indicating if only the balance changes should be // logged to the console. OnlyChanges bool + + // If non-empty, used to validate that /network/options matches the contents of the file + // located at this path. + asserterConfigurationPath string ) // rootPreRun is executed before the root command runs and sets up cpu @@ -197,7 +201,19 @@ default values.`, rootCmd.AddCommand(configurationValidateCmd) // Check commands + checkDataCmd.Flags().StringVar( + &asserterConfigurationPath, + "asserter-configuration-path", + "", // Default to skip validation + `Check that /network/options matches contents of file at this path`, + ) rootCmd.AddCommand(checkDataCmd) + checkConstructionCmd.Flags().StringVar( + &asserterConfigurationPath, + "asserter-configuration-path", + "", // Default to skip validation + `Check that /network/options matches contents of file at this path`, + ) rootCmd.AddCommand(checkConstructionCmd) // View Commands diff --git a/cmd/validate_asserter_config.go b/cmd/validate_asserter_config.go new file mode 100644 index 00000000..78fcc2e0 --- /dev/null +++ b/cmd/validate_asserter_config.go @@ -0,0 +1,162 @@ +package cmd + +import ( + "context" + "errors" + "fmt" + "github.com/coinbase/rosetta-sdk-go/asserter" + "github.com/coinbase/rosetta-sdk-go/fetcher" + "github.com/coinbase/rosetta-sdk-go/types" + "github.com/coinbase/rosetta-sdk-go/utils" + "reflect" + "sort" + "strings" +) + +// Confirm `/network/options` has relevant settings that match the assert configuration serlialized +// and written to `asserterConfigurationPath` +func validateAsserterConfiguration( + ctx context.Context, f *fetcher.Fetcher, network *types.NetworkIdentifier, + asserterConfigurationPath string, +) error { + resp, fetchErr := f.NetworkOptions(ctx, Config.Network, nil) + if fetchErr != nil { + return fetchErr.Err // how to return the errors? + } + + var asserterConfiguration asserter.Configuration + if err := utils.LoadAndParse(asserterConfigurationPath, &asserterConfiguration); err != nil { + return err + } + + return verifyAllowMatch(resp, &asserterConfiguration) +} + +func verifyAllowMatch( + resp *types.NetworkOptionsResponse, configuration *asserter.Configuration, +) error { + if resp == nil { + return errors.New("resp nil") + } + networkAllow := resp.Allow + if networkAllow == nil { + return errors.New("network's allow is nil") + } + if configuration == nil { + return errors.New("configuration nil") + } + + if err := verifyTimestampStartIndex( + networkAllow.TimestampStartIndex, configuration.AllowedTimestampStartIndex, + ); err != nil { + return err + } + + if err := verifyOperationTypes( + networkAllow.OperationTypes, configuration.AllowedOperationTypes, + ); err != nil { + return err + } + + if err := verifyOperationStatuses( + networkAllow.OperationStatuses, configuration.AllowedOperationStatuses, + ); err != nil { + return err + } + + return verifyErrors(networkAllow.Errors, configuration.AllowedErrors) +} + +func verifyTimestampStartIndex(networkTsi *int64, assertTsi int64) error { + if networkTsi == nil { + return errors.New("network allow timestamp start index nil") + } + if *networkTsi != assertTsi { + return fmt.Errorf( + "network and asserter timestamp start index mismatch. %d %d", + *networkTsi, assertTsi, + ) + } + + return nil +} + +func verifyOperationTypes(networkOt, asserterOt []string) error { + if len(networkOt) != len(asserterOt) { + return fmt.Errorf( + "network and asserter have operation type list size mismatch %v %v", + networkOt, asserterOt, + ) + } + + sort.Strings(networkOt) + sort.Strings(asserterOt) + + for i, networkOperationType := range networkOt { + asserterOperationType := asserterOt[i] + if networkOperationType != asserterOperationType { + return fmt.Errorf( + "network / asserter operation type mismatch %v %v", + networkOperationType, asserterOperationType, + ) + } + } + + return nil +} + +func verifyOperationStatuses(networkOs, asserterOs []*types.OperationStatus) error { + if len(networkOs) != len(asserterOs) { + return fmt.Errorf( + "network and asserter have operation status list size mismatch %v %v", + networkOs, asserterOs, + ) + } + + sort.Slice(networkOs, func(i, j int) bool { + return strings.Compare(networkOs[i].Status, networkOs[j].Status) < 0 + }) + sort.Slice(asserterOs, func(i, j int) bool { + return strings.Compare(asserterOs[i].Status, asserterOs[j].Status) < 0 + }) + + for i, networkOperationStatus := range networkOs { + asserterOperationStatus := asserterOs[i] + if !reflect.DeepEqual(networkOperationStatus, asserterOperationStatus) { + return fmt.Errorf( + "network / asserter operation status mismatch %v %v", + networkOperationStatus, asserterOperationStatus, + ) + } + } + + return nil +} + +func verifyErrors(networkErrors, asserterErrors []*types.Error) error { + if len(networkErrors) != len(asserterErrors) { + return fmt.Errorf( + "network and asserter have error list size mismatch %v %v", + networkErrors, asserterErrors, + ) + } + + sort.Slice(networkErrors, func(i, j int) bool { + return networkErrors[i].Code < networkErrors[j].Code + }) + sort.Slice(asserterErrors, func(i, j int) bool { + return asserterErrors[i].Code < asserterErrors[j].Code + }) + + for i, networkError := range networkErrors { + asserterError := asserterErrors[i] + if !reflect.DeepEqual(networkError, asserterError) { + return fmt.Errorf( + "network / asserter operation error mismatch %v %v", + networkError, asserterError, + ) + } + } + + return nil +} From a0cb09988a9b53dab1a019c6974b424cd1c923cb Mon Sep 17 00:00:00 2001 From: Swapna Gupta Date: Sat, 13 Mar 2021 21:47:58 -0800 Subject: [PATCH 252/376] make exempt accounts configuration relative --- .gitignore | 3 +++ configuration/configuration.go | 4 ++++ configuration/types.go | 2 ++ 3 files changed, 9 insertions(+) diff --git a/.gitignore b/.gitignore index 0891b757..40fb1133 100644 --- a/.gitignore +++ b/.gitignore @@ -3,3 +3,6 @@ bin/ # Intellij setting .idea/ + +# Mac files +.DS_Store diff --git a/configuration/configuration.go b/configuration/configuration.go index 969aee7f..2d9847f5 100644 --- a/configuration/configuration.go +++ b/configuration/configuration.go @@ -334,6 +334,10 @@ func modifyFilePaths(config *Configuration, fileDir string) { if len(config.Data.InterestingAccounts) > 0 { config.Data.InterestingAccounts = path.Join(fileDir, config.Data.InterestingAccounts) } + + if len(config.Data.ExemptAccounts) > 0 { + config.Data.ExemptAccounts = path.Join(fileDir, config.Data.ExemptAccounts) + } } if config.Construction != nil { diff --git a/configuration/types.go b/configuration/types.go index da4d1b2e..d88bedd4 100644 --- a/configuration/types.go +++ b/configuration/types.go @@ -351,6 +351,8 @@ type Configuration struct { OnlineURL string `json:"online_url"` // DataDirectory is a folder used to store logs and any data used to perform validation. + // The path can be absolute or relative to the directory containing the rosetta-cli + // binary. DataDirectory string `json:"data_directory"` // HTTPTimeout is the timeout for a HTTP request in seconds. From a8f90d6c534b4a90cc259ea98ff6f1577a9be67d Mon Sep 17 00:00:00 2001 From: Raghav Sharma Date: Mon, 15 Mar 2021 16:00:21 -0700 Subject: [PATCH 253/376] for construction too --- cmd/check_construction.go | 16 +++++++++++++++- cmd/validate_asserter_config.go | 21 +++++++++++---------- 2 files changed, 26 insertions(+), 11 deletions(-) diff --git a/cmd/check_construction.go b/cmd/check_construction.go index dc32d3ab..65e9bf1e 100644 --- a/cmd/check_construction.go +++ b/cmd/check_construction.go @@ -53,7 +53,7 @@ arbitrary scenarios (i.e. staking, governance).`, } ) -func runCheckConstructionCmd(cmd *cobra.Command, args []string) error { +func runCheckConstructionCmd(_ *cobra.Command, _ []string) error { if Config.Construction == nil { return results.ExitConstruction( Config, @@ -103,6 +103,20 @@ func runCheckConstructionCmd(cmd *cobra.Command, args []string) error { ) } + if asserterConfigurationPath != "" { + if err := validateAsserterConfiguration( + ctx, fetcher, Config.Network, asserterConfigurationPath, + ); err != nil { + cancel() + return results.ExitConstruction( + Config, + nil, + nil, + fmt.Errorf("%w: network options mismatch asserter configuration", err), + ) + } + } + constructionTester, err := tester.InitializeConstruction( ctx, Config, diff --git a/cmd/validate_asserter_config.go b/cmd/validate_asserter_config.go index 78fcc2e0..edd70605 100644 --- a/cmd/validate_asserter_config.go +++ b/cmd/validate_asserter_config.go @@ -13,15 +13,15 @@ import ( "strings" ) -// Confirm `/network/options` has relevant settings that match the assert configuration serlialized +// Confirm `/network/options` has relevant settings that match the assert configuration serialized // and written to `asserterConfigurationPath` func validateAsserterConfiguration( ctx context.Context, f *fetcher.Fetcher, network *types.NetworkIdentifier, asserterConfigurationPath string, ) error { - resp, fetchErr := f.NetworkOptions(ctx, Config.Network, nil) + resp, fetchErr := f.NetworkOptions(ctx, network, nil) if fetchErr != nil { - return fetchErr.Err // how to return the errors? + return fetchErr.Err } var asserterConfiguration asserter.Configuration @@ -32,8 +32,9 @@ func validateAsserterConfiguration( return verifyAllowMatch(resp, &asserterConfiguration) } +// Check the "allow" fields in the network options and asserter configuration match func verifyAllowMatch( - resp *types.NetworkOptionsResponse, configuration *asserter.Configuration, + resp *types.NetworkOptionsResponse, asserterConfig *asserter.Configuration, ) error { if resp == nil { return errors.New("resp nil") @@ -42,29 +43,29 @@ func verifyAllowMatch( if networkAllow == nil { return errors.New("network's allow is nil") } - if configuration == nil { - return errors.New("configuration nil") + if asserterConfig == nil { + return errors.New("asserterConfig nil") } if err := verifyTimestampStartIndex( - networkAllow.TimestampStartIndex, configuration.AllowedTimestampStartIndex, + networkAllow.TimestampStartIndex, asserterConfig.AllowedTimestampStartIndex, ); err != nil { return err } if err := verifyOperationTypes( - networkAllow.OperationTypes, configuration.AllowedOperationTypes, + networkAllow.OperationTypes, asserterConfig.AllowedOperationTypes, ); err != nil { return err } if err := verifyOperationStatuses( - networkAllow.OperationStatuses, configuration.AllowedOperationStatuses, + networkAllow.OperationStatuses, asserterConfig.AllowedOperationStatuses, ); err != nil { return err } - return verifyErrors(networkAllow.Errors, configuration.AllowedErrors) + return verifyErrors(networkAllow.Errors, asserterConfig.AllowedErrors) } func verifyTimestampStartIndex(networkTsi *int64, assertTsi int64) error { From d798ba2b85abe7fbe288dd727dbdc387d73dc8fb Mon Sep 17 00:00:00 2001 From: Raghav Sharma Date: Mon, 15 Mar 2021 16:19:22 -0700 Subject: [PATCH 254/376] Fix small bug --- asserterConfiguration.json | 172 ++++++++++++++++++++++++++++++++ cmd/validate_asserter_config.go | 4 +- 2 files changed, 174 insertions(+), 2 deletions(-) create mode 100644 asserterConfiguration.json diff --git a/asserterConfiguration.json b/asserterConfiguration.json new file mode 100644 index 00000000..f339bff6 --- /dev/null +++ b/asserterConfiguration.json @@ -0,0 +1,172 @@ +{ + "network_identifier": { + "blockchain": "cardano", + "network": "mainnet" + }, + "genesis_block_identifier": { + "index": 0, + "hash": "5f20df933584822601f9e3f8c024eb5eb252fe8cefb24d1317dc3d432e940ebb" + }, + "allowed_operation_types": [ + "input", + "output", + "stakeDelegation", + "stakeKeyDeregistration", + "stakeKeyRegistration", + "withdrawal" + ], + "allowed_operation_statuses": [ + { + "status": "success", + "successful": true + } + ], + "allowed_errors": [ + { + "code": 4001, + "message": "Block not found", + "retriable": false + }, + { + "code": 4002, + "message": "Network not found", + "retriable": false + }, + { + "code": 4003, + "message": "Networks not found", + "retriable": false + }, + { + "code": 4004, + "message": "Invalid blockchain", + "retriable": false + }, + { + "code": 4005, + "message": "Genesis block not found", + "retriable": false + }, + { + "code": 4006, + "message": "Transaction not found", + "retriable": false + }, + { + "code": 4007, + "message": "Invalid public key format", + "retriable": false + }, + { + "code": 4008, + "message": "Transaction inputs parameters errors in operations array", + "retriable": false + }, + { + "code": 4009, + "message": "Transaction outputs parameters errors in operations array", + "retriable": false + }, + { + "code": 4010, + "message": "The transaction you are trying to build has more outputs than inputs", + "retriable": false + }, + { + "code": 4011, + "message": "Cant create signed transaction from transaction bytes", + "retriable": false + }, + { + "code": 4012, + "message": "Cant create unsigned transaction from transaction bytes", + "retriable": false + }, + { + "code": 4013, + "message": "Cant deserialize transaction input from transaction body", + "retriable": false + }, + { + "code": 4014, + "message": "Cant deserialize transaction output from transaction body", + "retriable": false + }, + { + "code": 4015, + "message": "Provided address is invalid", + "retriable": true + }, + { + "code": 4016, + "message": "Provided address type is invalid", + "retriable": true + }, + { + "code": 4017, + "message": "Invalid staking key format", + "retriable": false + }, + { + "code": 4018, + "message": "Staking key is required for this type of address", + "retriable": false + }, + { + "code": 4019, + "message": "Provided operation type is invalid", + "retriable": true + }, + { + "code": 4020, + "message": "Pool key hash is required for stake delegation", + "retriable": false + }, + { + "code": 4021, + "message": "Assets are required for output operation token bundle", + "retriable": false + }, + { + "code": 4022, + "message": "Asset value is required for token asset", + "retriable": false + }, + { + "code": 5000, + "message": "An error occurred", + "retriable": true + }, + { + "code": 5001, + "message": "Not implemented", + "retriable": false + }, + { + "code": 5002, + "message": "Address generation error", + "retriable": false + }, + { + "code": 5003, + "message": "Parse signed transaction error", + "retriable": false + }, + { + "code": 5004, + "message": "Cant create signed transaction probably because of unsigned transaction bytes", + "retriable": false + }, + { + "code": 5005, + "message": "Cant build witnesses set for transaction probably because of provided signatures", + "retriable": false + }, + { + "code": 5006, + "message": "Error when sending the transaction", + "retriable": true + } + ], + "allowed_timestamp_start_index": 1 +} \ No newline at end of file diff --git a/cmd/validate_asserter_config.go b/cmd/validate_asserter_config.go index edd70605..f9b3a411 100644 --- a/cmd/validate_asserter_config.go +++ b/cmd/validate_asserter_config.go @@ -69,8 +69,8 @@ func verifyAllowMatch( } func verifyTimestampStartIndex(networkTsi *int64, assertTsi int64) error { - if networkTsi == nil { - return errors.New("network allow timestamp start index nil") + if networkTsi == nil { // This field is optional and defaults to all allowed + return nil } if *networkTsi != assertTsi { return fmt.Errorf( From 54e266e592cd26008ba3205a71062d439cc1c6e8 Mon Sep 17 00:00:00 2001 From: Raghav Sharma Date: Mon, 15 Mar 2021 16:21:03 -0700 Subject: [PATCH 255/376] rm config --- asserterConfiguration.json | 172 ------------------------------------- 1 file changed, 172 deletions(-) delete mode 100644 asserterConfiguration.json diff --git a/asserterConfiguration.json b/asserterConfiguration.json deleted file mode 100644 index f339bff6..00000000 --- a/asserterConfiguration.json +++ /dev/null @@ -1,172 +0,0 @@ -{ - "network_identifier": { - "blockchain": "cardano", - "network": "mainnet" - }, - "genesis_block_identifier": { - "index": 0, - "hash": "5f20df933584822601f9e3f8c024eb5eb252fe8cefb24d1317dc3d432e940ebb" - }, - "allowed_operation_types": [ - "input", - "output", - "stakeDelegation", - "stakeKeyDeregistration", - "stakeKeyRegistration", - "withdrawal" - ], - "allowed_operation_statuses": [ - { - "status": "success", - "successful": true - } - ], - "allowed_errors": [ - { - "code": 4001, - "message": "Block not found", - "retriable": false - }, - { - "code": 4002, - "message": "Network not found", - "retriable": false - }, - { - "code": 4003, - "message": "Networks not found", - "retriable": false - }, - { - "code": 4004, - "message": "Invalid blockchain", - "retriable": false - }, - { - "code": 4005, - "message": "Genesis block not found", - "retriable": false - }, - { - "code": 4006, - "message": "Transaction not found", - "retriable": false - }, - { - "code": 4007, - "message": "Invalid public key format", - "retriable": false - }, - { - "code": 4008, - "message": "Transaction inputs parameters errors in operations array", - "retriable": false - }, - { - "code": 4009, - "message": "Transaction outputs parameters errors in operations array", - "retriable": false - }, - { - "code": 4010, - "message": "The transaction you are trying to build has more outputs than inputs", - "retriable": false - }, - { - "code": 4011, - "message": "Cant create signed transaction from transaction bytes", - "retriable": false - }, - { - "code": 4012, - "message": "Cant create unsigned transaction from transaction bytes", - "retriable": false - }, - { - "code": 4013, - "message": "Cant deserialize transaction input from transaction body", - "retriable": false - }, - { - "code": 4014, - "message": "Cant deserialize transaction output from transaction body", - "retriable": false - }, - { - "code": 4015, - "message": "Provided address is invalid", - "retriable": true - }, - { - "code": 4016, - "message": "Provided address type is invalid", - "retriable": true - }, - { - "code": 4017, - "message": "Invalid staking key format", - "retriable": false - }, - { - "code": 4018, - "message": "Staking key is required for this type of address", - "retriable": false - }, - { - "code": 4019, - "message": "Provided operation type is invalid", - "retriable": true - }, - { - "code": 4020, - "message": "Pool key hash is required for stake delegation", - "retriable": false - }, - { - "code": 4021, - "message": "Assets are required for output operation token bundle", - "retriable": false - }, - { - "code": 4022, - "message": "Asset value is required for token asset", - "retriable": false - }, - { - "code": 5000, - "message": "An error occurred", - "retriable": true - }, - { - "code": 5001, - "message": "Not implemented", - "retriable": false - }, - { - "code": 5002, - "message": "Address generation error", - "retriable": false - }, - { - "code": 5003, - "message": "Parse signed transaction error", - "retriable": false - }, - { - "code": 5004, - "message": "Cant create signed transaction probably because of unsigned transaction bytes", - "retriable": false - }, - { - "code": 5005, - "message": "Cant build witnesses set for transaction probably because of provided signatures", - "retriable": false - }, - { - "code": 5006, - "message": "Error when sending the transaction", - "retriable": true - } - ], - "allowed_timestamp_start_index": 1 -} \ No newline at end of file From 4ca419d7668ef928b3b6815fc6976e571c11f068 Mon Sep 17 00:00:00 2001 From: Raghav Sharma Date: Mon, 15 Mar 2021 20:16:12 -0700 Subject: [PATCH 256/376] clean --- cmd/check_construction.go | 20 ++++++++-- cmd/check_data.go | 22 +++++++++-- cmd/root.go | 10 ++--- cmd/validate_asserter_config.go | 66 +++++++++++++++++++-------------- 4 files changed, 78 insertions(+), 40 deletions(-) diff --git a/cmd/check_construction.go b/cmd/check_construction.go index 65e9bf1e..91cc11ec 100644 --- a/cmd/check_construction.go +++ b/cmd/check_construction.go @@ -103,16 +103,28 @@ func runCheckConstructionCmd(_ *cobra.Command, _ []string) error { ) } - if asserterConfigurationPath != "" { - if err := validateAsserterConfiguration( - ctx, fetcher, Config.Network, asserterConfigurationPath, + if asserterConfigurationFile != "" { + networkAllow, asserterConfiguration, err := getNetworkAllowAndAsserterConfiguration( + ctx, fetcher, Config.Network, asserterConfigurationFile) + if err != nil { + cancel() + return results.ExitConstruction( + Config, + nil, + nil, + fmt.Errorf("%w: failure getting /network/options or asserter configuration", err), + ) + } + + if err := validateNetworkAndAsserterAllowMatch( + networkAllow, asserterConfiguration, ); err != nil { cancel() return results.ExitConstruction( Config, nil, nil, - fmt.Errorf("%w: network options mismatch asserter configuration", err), + fmt.Errorf("%w: network / asserter configuration validation failure", err), ) } } diff --git a/cmd/check_data.go b/cmd/check_data.go index ecbbd131..d1e831e8 100644 --- a/cmd/check_data.go +++ b/cmd/check_data.go @@ -115,16 +115,30 @@ func runCheckDataCmd(_ *cobra.Command, _ []string) error { ) } - if asserterConfigurationPath != "" { - if err := validateAsserterConfiguration( - ctx, fetcher, Config.Network, asserterConfigurationPath, + if asserterConfigurationFile != "" { + networkAllow, asserterConfiguration, err := getNetworkAllowAndAsserterConfiguration( + ctx, fetcher, Config.Network, asserterConfigurationFile) + if err != nil { + cancel() + return results.ExitData( + Config, + nil, + nil, + fmt.Errorf("%w: failure getting /network/options or asserter configuration", err), + "", + "", + ) + } + + if err := validateNetworkAndAsserterAllowMatch( + networkAllow, asserterConfiguration, ); err != nil { cancel() return results.ExitData( Config, nil, nil, - fmt.Errorf("%w: network options mismatch asserter configuration", err), + fmt.Errorf("%w: network / asserter configuration validation failure", err), "", "", ) diff --git a/cmd/root.go b/cmd/root.go index 5b8b13aa..f408f4b0 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -74,7 +74,7 @@ var ( // If non-empty, used to validate that /network/options matches the contents of the file // located at this path. - asserterConfigurationPath string + asserterConfigurationFile string ) // rootPreRun is executed before the root command runs and sets up cpu @@ -202,15 +202,15 @@ default values.`, // Check commands checkDataCmd.Flags().StringVar( - &asserterConfigurationPath, - "asserter-configuration-path", + &asserterConfigurationFile, + "asserter-configuration-file", "", // Default to skip validation `Check that /network/options matches contents of file at this path`, ) rootCmd.AddCommand(checkDataCmd) checkConstructionCmd.Flags().StringVar( - &asserterConfigurationPath, - "asserter-configuration-path", + &asserterConfigurationFile, + "asserter-configuration-file", "", // Default to skip validation `Check that /network/options matches contents of file at this path`, ) diff --git a/cmd/validate_asserter_config.go b/cmd/validate_asserter_config.go index f9b3a411..7b4fe735 100644 --- a/cmd/validate_asserter_config.go +++ b/cmd/validate_asserter_config.go @@ -1,3 +1,17 @@ +// Copyright 2020 Coinbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package cmd import ( @@ -13,59 +27,57 @@ import ( "strings" ) -// Confirm `/network/options` has relevant settings that match the assert configuration serialized -// and written to `asserterConfigurationPath` -func validateAsserterConfiguration( +// Common helper across Construction and Data +// Issues an RPC to fetch /network/options, and extracts the `Allow` +// Reads the JSON file at `asserterConfigurationFile` and loads into a Go object +// Intended to feed directly into `validateNetworkAndAsserterAllowMatch` +// This is a standalone function so the latter can be easily unit tested +func getNetworkAllowAndAsserterConfiguration( ctx context.Context, f *fetcher.Fetcher, network *types.NetworkIdentifier, - asserterConfigurationPath string, -) error { - resp, fetchErr := f.NetworkOptions(ctx, network, nil) - if fetchErr != nil { - return fetchErr.Err + asserterConfigurationFile string, +) (*types.Allow, *asserter.Configuration, error) { + var asserterConfiguration asserter.Configuration + if err := utils.LoadAndParse(asserterConfigurationFile, &asserterConfiguration); err != nil { + return nil, nil, err } - var asserterConfiguration asserter.Configuration - if err := utils.LoadAndParse(asserterConfigurationPath, &asserterConfiguration); err != nil { - return err + resp, fetchErr := f.NetworkOptions(ctx, network, nil) + if fetchErr != nil { + return nil, nil, fetchErr.Err } - return verifyAllowMatch(resp, &asserterConfiguration) + return resp.Allow, &asserterConfiguration, nil } -// Check the "allow" fields in the network options and asserter configuration match -func verifyAllowMatch( - resp *types.NetworkOptionsResponse, asserterConfig *asserter.Configuration, +func validateNetworkAndAsserterAllowMatch( + networkAllow *types.Allow, asserterConfiguration *asserter.Configuration, ) error { - if resp == nil { - return errors.New("resp nil") - } - networkAllow := resp.Allow if networkAllow == nil { - return errors.New("network's allow is nil") + return errors.New("networkAllow nil") } - if asserterConfig == nil { - return errors.New("asserterConfig nil") + if asserterConfiguration == nil { + return errors.New("asserterConfiguration nil") } if err := verifyTimestampStartIndex( - networkAllow.TimestampStartIndex, asserterConfig.AllowedTimestampStartIndex, + networkAllow.TimestampStartIndex, asserterConfiguration.AllowedTimestampStartIndex, ); err != nil { return err } if err := verifyOperationTypes( - networkAllow.OperationTypes, asserterConfig.AllowedOperationTypes, + networkAllow.OperationTypes, asserterConfiguration.AllowedOperationTypes, ); err != nil { return err } if err := verifyOperationStatuses( - networkAllow.OperationStatuses, asserterConfig.AllowedOperationStatuses, + networkAllow.OperationStatuses, asserterConfiguration.AllowedOperationStatuses, ); err != nil { return err } - return verifyErrors(networkAllow.Errors, asserterConfig.AllowedErrors) + return verifyErrors(networkAllow.Errors, asserterConfiguration.AllowedErrors) } func verifyTimestampStartIndex(networkTsi *int64, assertTsi int64) error { @@ -153,7 +165,7 @@ func verifyErrors(networkErrors, asserterErrors []*types.Error) error { asserterError := asserterErrors[i] if !reflect.DeepEqual(networkError, asserterError) { return fmt.Errorf( - "network / asserter operation error mismatch %v %v", + "network / asserter error mismatch %v %v", networkError, asserterError, ) } From cc70bb52fa9ff5c9bec9b4930480c89b0b0b59fc Mon Sep 17 00:00:00 2001 From: Raghav Sharma Date: Mon, 15 Mar 2021 20:33:54 -0700 Subject: [PATCH 257/376] unit test --- cmd/validate_asserter_config_test.go | 69 ++++++++++++++++++++++++++++ 1 file changed, 69 insertions(+) create mode 100644 cmd/validate_asserter_config_test.go diff --git a/cmd/validate_asserter_config_test.go b/cmd/validate_asserter_config_test.go new file mode 100644 index 00000000..5c0ec4e4 --- /dev/null +++ b/cmd/validate_asserter_config_test.go @@ -0,0 +1,69 @@ +// Copyright 2020 Coinbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package cmd + +import ( + "github.com/coinbase/rosetta-sdk-go/asserter" + "github.com/coinbase/rosetta-sdk-go/types" + "testing" + "github.com/stretchr/testify/assert" +) + +func TestMatch(t *testing.T) { + networkAllow, asserterConfiguration := generateNetworkAllowAndAsserterConfiguration() + assert.NoError(t, validateNetworkAndAsserterAllowMatch(networkAllow, asserterConfiguration)) +} + +func generateNetworkAllowAndAsserterConfiguration() ( + *types.Allow, *asserter.Configuration, +) { + operationStatuses := []*types.OperationStatus{ + { + Successful: true, + Status: "status0", + }, + { + // Successful: false + Status: "status1", + }, + } + operationTypes := []string{"type0", "type1"} + errors := []*types.Error{ + { + Code: 1, + Message: "message1", + }, + { + Code: 2, + Message: "message2", + }, + } + var tsi int64 = 5 + + allow := &types.Allow{ + OperationStatuses: operationStatuses, + OperationTypes: operationTypes, + Errors: errors, + TimestampStartIndex: &tsi, + } + config := &asserter.Configuration{ + AllowedOperationStatuses: operationStatuses, + AllowedOperationTypes: operationTypes, + AllowedErrors: errors, + AllowedTimestampStartIndex: tsi, + } + + return allow, config +} From ed074e8c706987a2c385ed463f851028252278ae Mon Sep 17 00:00:00 2001 From: Raghav Sharma Date: Mon, 15 Mar 2021 20:51:25 -0700 Subject: [PATCH 258/376] Test progress --- cmd/validate_asserter_config_test.go | 52 ++++++++++++++++++++++++++-- 1 file changed, 50 insertions(+), 2 deletions(-) diff --git a/cmd/validate_asserter_config_test.go b/cmd/validate_asserter_config_test.go index 5c0ec4e4..a583ab45 100644 --- a/cmd/validate_asserter_config_test.go +++ b/cmd/validate_asserter_config_test.go @@ -23,9 +23,41 @@ import ( func TestMatch(t *testing.T) { networkAllow, asserterConfiguration := generateNetworkAllowAndAsserterConfiguration() - assert.NoError(t, validateNetworkAndAsserterAllowMatch(networkAllow, asserterConfiguration)) + confirmSuccess(t, networkAllow, asserterConfiguration) +} + +func TestNil(t *testing.T) { + networkAllow, asserterConfiguration := generateNetworkAllowAndAsserterConfiguration() + confirmError(t, nil, asserterConfiguration) + confirmError(t, networkAllow, nil) } +func TestTsi(t *testing.T) { + networkAllow, asserterConfiguration := generateNetworkAllowAndAsserterConfiguration() + networkAllow.TimestampStartIndex = nil + confirmSuccess(t, networkAllow, asserterConfiguration) + + networkAllow, asserterConfiguration = generateNetworkAllowAndAsserterConfiguration() + asserterConfiguration.AllowedTimestampStartIndex = 567 + confirmError(t, networkAllow, asserterConfiguration) +} + +func TestOperationTypes(t *testing.T) { + networkAllow, asserterConfiguration := generateNetworkAllowAndAsserterConfiguration() + networkAllow.OperationTypes = generateOperationTypes() + networkAllow.OperationTypes[1] = "mismatchType" + confirmError(t, networkAllow, asserterConfiguration) + + networkAllow, _ = generateNetworkAllowAndAsserterConfiguration() + networkAllow.OperationTypes = append(networkAllow.OperationTypes, "extra") + confirmError(t, networkAllow, asserterConfiguration) + + networkAllow, _ = generateNetworkAllowAndAsserterConfiguration() + asserterConfiguration.AllowedOperationTypes = nil + confirmError(t, networkAllow, asserterConfiguration) +} + +// Generate simple configs for testing func generateNetworkAllowAndAsserterConfiguration() ( *types.Allow, *asserter.Configuration, ) { @@ -39,7 +71,7 @@ func generateNetworkAllowAndAsserterConfiguration() ( Status: "status1", }, } - operationTypes := []string{"type0", "type1"} + operationTypes := generateOperationTypes() errors := []*types.Error{ { Code: 1, @@ -67,3 +99,19 @@ func generateNetworkAllowAndAsserterConfiguration() ( return allow, config } + +func generateOperationTypes() []string { + return []string{"type0", "type1"} +} + +func confirmSuccess( + t *testing.T, networkAllow *types.Allow, asserterConfiguration *asserter.Configuration, +) { + assert.NoError(t, validateNetworkAndAsserterAllowMatch(networkAllow, asserterConfiguration)) +} + +func confirmError( + t *testing.T, networkAllow *types.Allow, asserterConfiguration *asserter.Configuration, +) { + assert.Error(t, validateNetworkAndAsserterAllowMatch(networkAllow, asserterConfiguration)) +} \ No newline at end of file From a39e2983c1581b7f1aec0c59565bbb9e20152f6e Mon Sep 17 00:00:00 2001 From: Raghav Sharma Date: Mon, 15 Mar 2021 21:01:36 -0700 Subject: [PATCH 259/376] Status test --- cmd/validate_asserter_config_test.go | 74 ++++++++++++++++++---------- 1 file changed, 48 insertions(+), 26 deletions(-) diff --git a/cmd/validate_asserter_config_test.go b/cmd/validate_asserter_config_test.go index a583ab45..e1e894fa 100644 --- a/cmd/validate_asserter_config_test.go +++ b/cmd/validate_asserter_config_test.go @@ -44,12 +44,10 @@ func TestTsi(t *testing.T) { func TestOperationTypes(t *testing.T) { networkAllow, asserterConfiguration := generateNetworkAllowAndAsserterConfiguration() - networkAllow.OperationTypes = generateOperationTypes() networkAllow.OperationTypes[1] = "mismatchType" confirmError(t, networkAllow, asserterConfiguration) - networkAllow, _ = generateNetworkAllowAndAsserterConfiguration() - networkAllow.OperationTypes = append(networkAllow.OperationTypes, "extra") + networkAllow.OperationTypes = append(generateOperationTypes(), "extra") confirmError(t, networkAllow, asserterConfiguration) networkAllow, _ = generateNetworkAllowAndAsserterConfiguration() @@ -57,11 +55,53 @@ func TestOperationTypes(t *testing.T) { confirmError(t, networkAllow, asserterConfiguration) } +func TestOperationStatuses(t *testing.T) { + networkAllow, asserterConfiguration := generateNetworkAllowAndAsserterConfiguration() + networkAllow.OperationStatuses[0].Successful = !networkAllow.OperationStatuses[0].Successful + confirmError(t, networkAllow, asserterConfiguration) + + networkAllow, _ = generateNetworkAllowAndAsserterConfiguration() + asserterConfiguration.AllowedOperationStatuses[1].Status = "mismatchStatus" + confirmError(t, networkAllow, asserterConfiguration) + + _, asserterConfiguration = generateNetworkAllowAndAsserterConfiguration() + asserterConfiguration.AllowedOperationStatuses = append(generateOperationStatuses(), + &types.OperationStatus{Status: "extra"}) + confirmError(t, networkAllow, asserterConfiguration) + + _, asserterConfiguration = generateNetworkAllowAndAsserterConfiguration() + networkAllow.OperationStatuses = nil + confirmError(t, networkAllow, asserterConfiguration) +} + // Generate simple configs for testing +// Generators used internally below are so they are logically equal but can be mutated separately func generateNetworkAllowAndAsserterConfiguration() ( *types.Allow, *asserter.Configuration, ) { - operationStatuses := []*types.OperationStatus{ + var tsi int64 = 5 + allow := &types.Allow{ + OperationStatuses: generateOperationStatuses(), + OperationTypes: generateOperationTypes(), + Errors: generateErrors(), + TimestampStartIndex: &tsi, + } + config := &asserter.Configuration{ + AllowedOperationStatuses: generateOperationStatuses(), + AllowedOperationTypes: generateOperationTypes(), + AllowedErrors: generateErrors(), + AllowedTimestampStartIndex: tsi, + } + + return allow, config +} + +func generateOperationTypes() []string { + return []string{"type0", "type1"} +} + +func generateOperationStatuses() []*types.OperationStatus { + return []*types.OperationStatus{ { Successful: true, Status: "status0", @@ -71,8 +111,10 @@ func generateNetworkAllowAndAsserterConfiguration() ( Status: "status1", }, } - operationTypes := generateOperationTypes() - errors := []*types.Error{ +} + +func generateErrors() []*types.Error { + return []*types.Error{ { Code: 1, Message: "message1", @@ -82,26 +124,6 @@ func generateNetworkAllowAndAsserterConfiguration() ( Message: "message2", }, } - var tsi int64 = 5 - - allow := &types.Allow{ - OperationStatuses: operationStatuses, - OperationTypes: operationTypes, - Errors: errors, - TimestampStartIndex: &tsi, - } - config := &asserter.Configuration{ - AllowedOperationStatuses: operationStatuses, - AllowedOperationTypes: operationTypes, - AllowedErrors: errors, - AllowedTimestampStartIndex: tsi, - } - - return allow, config -} - -func generateOperationTypes() []string { - return []string{"type0", "type1"} } func confirmSuccess( From 4dfb8b56bdac2f0023923197bccc19de79b036ce Mon Sep 17 00:00:00 2001 From: Raghav Sharma Date: Mon, 15 Mar 2021 21:07:52 -0700 Subject: [PATCH 260/376] finish unit test --- cmd/validate_asserter_config_test.go | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/cmd/validate_asserter_config_test.go b/cmd/validate_asserter_config_test.go index e1e894fa..f44f006a 100644 --- a/cmd/validate_asserter_config_test.go +++ b/cmd/validate_asserter_config_test.go @@ -74,6 +74,26 @@ func TestOperationStatuses(t *testing.T) { confirmError(t, networkAllow, asserterConfiguration) } +func TestErrors(t *testing.T) { + networkAllow, asserterConfiguration := generateNetworkAllowAndAsserterConfiguration() + networkAllow.Errors[0].Code = 123 + confirmError(t, networkAllow, asserterConfiguration) + + networkAllow, _ = generateNetworkAllowAndAsserterConfiguration() + asserterConfiguration.AllowedErrors[1].Message = "mismatchMessage" + confirmError(t, networkAllow, asserterConfiguration) + + _, asserterConfiguration = generateNetworkAllowAndAsserterConfiguration() + networkAllow.Errors[0].Details = map[string]interface{}{"key": "value"} + asserterConfiguration.AllowedErrors[0].Details = map[string]interface{}{"key": "differentValue"} + confirmError(t, networkAllow, asserterConfiguration) + + networkAllow, asserterConfiguration = generateNetworkAllowAndAsserterConfiguration() + asserterConfiguration.AllowedErrors = append(asserterConfiguration.AllowedErrors, + &types.Error{Code: 123, Message: "extra"}) + confirmError(t, networkAllow, asserterConfiguration) +} + // Generate simple configs for testing // Generators used internally below are so they are logically equal but can be mutated separately func generateNetworkAllowAndAsserterConfiguration() ( From 4bc5c7ab4b49339723b781094146ee04ff93ae74 Mon Sep 17 00:00:00 2001 From: Raghav Sharma Date: Mon, 15 Mar 2021 21:11:43 -0700 Subject: [PATCH 261/376] add newline --- cmd/validate_asserter_config_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/validate_asserter_config_test.go b/cmd/validate_asserter_config_test.go index f44f006a..12d2c24f 100644 --- a/cmd/validate_asserter_config_test.go +++ b/cmd/validate_asserter_config_test.go @@ -156,4 +156,4 @@ func confirmError( t *testing.T, networkAllow *types.Allow, asserterConfiguration *asserter.Configuration, ) { assert.Error(t, validateNetworkAndAsserterAllowMatch(networkAllow, asserterConfiguration)) -} \ No newline at end of file +} From 270c59cdb8e4ed1942b86b7ab8dce6e710b9bc29 Mon Sep 17 00:00:00 2001 From: Swapna Gupta Date: Tue, 16 Mar 2021 17:16:29 -0700 Subject: [PATCH 262/376] fix description --- configuration/types.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/configuration/types.go b/configuration/types.go index d88bedd4..13ee43e6 100644 --- a/configuration/types.go +++ b/configuration/types.go @@ -351,8 +351,8 @@ type Configuration struct { OnlineURL string `json:"online_url"` // DataDirectory is a folder used to store logs and any data used to perform validation. - // The path can be absolute or relative to the directory containing the rosetta-cli - // binary. + // The path can be absolute, or it can be relative to where rosetta-cli + // binary is being executed. DataDirectory string `json:"data_directory"` // HTTPTimeout is the timeout for a HTTP request in seconds. From bcc1175d22dcfc517d284fb9d95d0c63219dd838 Mon Sep 17 00:00:00 2001 From: Raghav Sharma Date: Tue, 16 Mar 2021 17:58:04 -0700 Subject: [PATCH 263/376] update comment --- asserterConfiguration.json | 172 +++++++++++++++++++++++++++++++++++++ cmd/root.go | 11 ++- 2 files changed, 182 insertions(+), 1 deletion(-) create mode 100644 asserterConfiguration.json diff --git a/asserterConfiguration.json b/asserterConfiguration.json new file mode 100644 index 00000000..f339bff6 --- /dev/null +++ b/asserterConfiguration.json @@ -0,0 +1,172 @@ +{ + "network_identifier": { + "blockchain": "cardano", + "network": "mainnet" + }, + "genesis_block_identifier": { + "index": 0, + "hash": "5f20df933584822601f9e3f8c024eb5eb252fe8cefb24d1317dc3d432e940ebb" + }, + "allowed_operation_types": [ + "input", + "output", + "stakeDelegation", + "stakeKeyDeregistration", + "stakeKeyRegistration", + "withdrawal" + ], + "allowed_operation_statuses": [ + { + "status": "success", + "successful": true + } + ], + "allowed_errors": [ + { + "code": 4001, + "message": "Block not found", + "retriable": false + }, + { + "code": 4002, + "message": "Network not found", + "retriable": false + }, + { + "code": 4003, + "message": "Networks not found", + "retriable": false + }, + { + "code": 4004, + "message": "Invalid blockchain", + "retriable": false + }, + { + "code": 4005, + "message": "Genesis block not found", + "retriable": false + }, + { + "code": 4006, + "message": "Transaction not found", + "retriable": false + }, + { + "code": 4007, + "message": "Invalid public key format", + "retriable": false + }, + { + "code": 4008, + "message": "Transaction inputs parameters errors in operations array", + "retriable": false + }, + { + "code": 4009, + "message": "Transaction outputs parameters errors in operations array", + "retriable": false + }, + { + "code": 4010, + "message": "The transaction you are trying to build has more outputs than inputs", + "retriable": false + }, + { + "code": 4011, + "message": "Cant create signed transaction from transaction bytes", + "retriable": false + }, + { + "code": 4012, + "message": "Cant create unsigned transaction from transaction bytes", + "retriable": false + }, + { + "code": 4013, + "message": "Cant deserialize transaction input from transaction body", + "retriable": false + }, + { + "code": 4014, + "message": "Cant deserialize transaction output from transaction body", + "retriable": false + }, + { + "code": 4015, + "message": "Provided address is invalid", + "retriable": true + }, + { + "code": 4016, + "message": "Provided address type is invalid", + "retriable": true + }, + { + "code": 4017, + "message": "Invalid staking key format", + "retriable": false + }, + { + "code": 4018, + "message": "Staking key is required for this type of address", + "retriable": false + }, + { + "code": 4019, + "message": "Provided operation type is invalid", + "retriable": true + }, + { + "code": 4020, + "message": "Pool key hash is required for stake delegation", + "retriable": false + }, + { + "code": 4021, + "message": "Assets are required for output operation token bundle", + "retriable": false + }, + { + "code": 4022, + "message": "Asset value is required for token asset", + "retriable": false + }, + { + "code": 5000, + "message": "An error occurred", + "retriable": true + }, + { + "code": 5001, + "message": "Not implemented", + "retriable": false + }, + { + "code": 5002, + "message": "Address generation error", + "retriable": false + }, + { + "code": 5003, + "message": "Parse signed transaction error", + "retriable": false + }, + { + "code": 5004, + "message": "Cant create signed transaction probably because of unsigned transaction bytes", + "retriable": false + }, + { + "code": 5005, + "message": "Cant build witnesses set for transaction probably because of provided signatures", + "retriable": false + }, + { + "code": 5006, + "message": "Error when sending the transaction", + "retriable": true + } + ], + "allowed_timestamp_start_index": 1 +} \ No newline at end of file diff --git a/cmd/root.go b/cmd/root.go index f408f4b0..130ee0d3 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -73,7 +73,16 @@ var ( OnlyChanges bool // If non-empty, used to validate that /network/options matches the contents of the file - // located at this path. + // located at this path. The intended use case is someone previously ran + // utils:asserter-configuration `asserterConfigurationFile`, so the validation is being done + // against the same file (or an identical copy of that file at this path). This file contains a + // snapshot of /network/options (and some other unrelated configuration options), so this will + // trigger validation of the current /network/options against that. See + // utils:asserter-configuration for more details. + // + // The main goal is to verify these core configuration options don't change across releases, + // which has caused production incidents in the past. This can be used for both check:data + // and check:construction. asserterConfigurationFile string ) From f45f67fcb89b1a42217dc4dba207332fc8461785 Mon Sep 17 00:00:00 2001 From: Raghav Sharma Date: Tue, 16 Mar 2021 18:12:26 -0700 Subject: [PATCH 264/376] better errors, refactor --- asserterConfiguration.json | 172 -------------------------------- cmd/check_construction.go | 18 +--- cmd/check_data.go | 20 +--- cmd/validate_asserter_config.go | 32 +++--- 4 files changed, 22 insertions(+), 220 deletions(-) delete mode 100644 asserterConfiguration.json diff --git a/asserterConfiguration.json b/asserterConfiguration.json deleted file mode 100644 index f339bff6..00000000 --- a/asserterConfiguration.json +++ /dev/null @@ -1,172 +0,0 @@ -{ - "network_identifier": { - "blockchain": "cardano", - "network": "mainnet" - }, - "genesis_block_identifier": { - "index": 0, - "hash": "5f20df933584822601f9e3f8c024eb5eb252fe8cefb24d1317dc3d432e940ebb" - }, - "allowed_operation_types": [ - "input", - "output", - "stakeDelegation", - "stakeKeyDeregistration", - "stakeKeyRegistration", - "withdrawal" - ], - "allowed_operation_statuses": [ - { - "status": "success", - "successful": true - } - ], - "allowed_errors": [ - { - "code": 4001, - "message": "Block not found", - "retriable": false - }, - { - "code": 4002, - "message": "Network not found", - "retriable": false - }, - { - "code": 4003, - "message": "Networks not found", - "retriable": false - }, - { - "code": 4004, - "message": "Invalid blockchain", - "retriable": false - }, - { - "code": 4005, - "message": "Genesis block not found", - "retriable": false - }, - { - "code": 4006, - "message": "Transaction not found", - "retriable": false - }, - { - "code": 4007, - "message": "Invalid public key format", - "retriable": false - }, - { - "code": 4008, - "message": "Transaction inputs parameters errors in operations array", - "retriable": false - }, - { - "code": 4009, - "message": "Transaction outputs parameters errors in operations array", - "retriable": false - }, - { - "code": 4010, - "message": "The transaction you are trying to build has more outputs than inputs", - "retriable": false - }, - { - "code": 4011, - "message": "Cant create signed transaction from transaction bytes", - "retriable": false - }, - { - "code": 4012, - "message": "Cant create unsigned transaction from transaction bytes", - "retriable": false - }, - { - "code": 4013, - "message": "Cant deserialize transaction input from transaction body", - "retriable": false - }, - { - "code": 4014, - "message": "Cant deserialize transaction output from transaction body", - "retriable": false - }, - { - "code": 4015, - "message": "Provided address is invalid", - "retriable": true - }, - { - "code": 4016, - "message": "Provided address type is invalid", - "retriable": true - }, - { - "code": 4017, - "message": "Invalid staking key format", - "retriable": false - }, - { - "code": 4018, - "message": "Staking key is required for this type of address", - "retriable": false - }, - { - "code": 4019, - "message": "Provided operation type is invalid", - "retriable": true - }, - { - "code": 4020, - "message": "Pool key hash is required for stake delegation", - "retriable": false - }, - { - "code": 4021, - "message": "Assets are required for output operation token bundle", - "retriable": false - }, - { - "code": 4022, - "message": "Asset value is required for token asset", - "retriable": false - }, - { - "code": 5000, - "message": "An error occurred", - "retriable": true - }, - { - "code": 5001, - "message": "Not implemented", - "retriable": false - }, - { - "code": 5002, - "message": "Address generation error", - "retriable": false - }, - { - "code": 5003, - "message": "Parse signed transaction error", - "retriable": false - }, - { - "code": 5004, - "message": "Cant create signed transaction probably because of unsigned transaction bytes", - "retriable": false - }, - { - "code": 5005, - "message": "Cant build witnesses set for transaction probably because of provided signatures", - "retriable": false - }, - { - "code": 5006, - "message": "Error when sending the transaction", - "retriable": true - } - ], - "allowed_timestamp_start_index": 1 -} \ No newline at end of file diff --git a/cmd/check_construction.go b/cmd/check_construction.go index 91cc11ec..4a0febf7 100644 --- a/cmd/check_construction.go +++ b/cmd/check_construction.go @@ -104,27 +104,15 @@ func runCheckConstructionCmd(_ *cobra.Command, _ []string) error { } if asserterConfigurationFile != "" { - networkAllow, asserterConfiguration, err := getNetworkAllowAndAsserterConfiguration( - ctx, fetcher, Config.Network, asserterConfigurationFile) - if err != nil { - cancel() - return results.ExitConstruction( - Config, - nil, - nil, - fmt.Errorf("%w: failure getting /network/options or asserter configuration", err), - ) - } - - if err := validateNetworkAndAsserterAllowMatch( - networkAllow, asserterConfiguration, + if err := validateNetworkOptionsMatchesAsserterConfiguration( + ctx, fetcher, Config.Network, asserterConfigurationFile, ); err != nil { cancel() return results.ExitConstruction( Config, nil, nil, - fmt.Errorf("%w: network / asserter configuration validation failure", err), + err, ) } } diff --git a/cmd/check_data.go b/cmd/check_data.go index d1e831e8..46eb5dbc 100644 --- a/cmd/check_data.go +++ b/cmd/check_data.go @@ -116,29 +116,15 @@ func runCheckDataCmd(_ *cobra.Command, _ []string) error { } if asserterConfigurationFile != "" { - networkAllow, asserterConfiguration, err := getNetworkAllowAndAsserterConfiguration( - ctx, fetcher, Config.Network, asserterConfigurationFile) - if err != nil { - cancel() - return results.ExitData( - Config, - nil, - nil, - fmt.Errorf("%w: failure getting /network/options or asserter configuration", err), - "", - "", - ) - } - - if err := validateNetworkAndAsserterAllowMatch( - networkAllow, asserterConfiguration, + if err := validateNetworkOptionsMatchesAsserterConfiguration( + ctx, fetcher, Config.Network, asserterConfigurationFile, ); err != nil { cancel() return results.ExitData( Config, nil, nil, - fmt.Errorf("%w: network / asserter configuration validation failure", err), + err, "", "", ) diff --git a/cmd/validate_asserter_config.go b/cmd/validate_asserter_config.go index 7b4fe735..5e3be606 100644 --- a/cmd/validate_asserter_config.go +++ b/cmd/validate_asserter_config.go @@ -30,33 +30,32 @@ import ( // Common helper across Construction and Data // Issues an RPC to fetch /network/options, and extracts the `Allow` // Reads the JSON file at `asserterConfigurationFile` and loads into a Go object -// Intended to feed directly into `validateNetworkAndAsserterAllowMatch` -// This is a standalone function so the latter can be easily unit tested -func getNetworkAllowAndAsserterConfiguration( +// Validates the `Allow`s across both objects match +func validateNetworkOptionsMatchesAsserterConfiguration( ctx context.Context, f *fetcher.Fetcher, network *types.NetworkIdentifier, asserterConfigurationFile string, -) (*types.Allow, *asserter.Configuration, error) { +) error { var asserterConfiguration asserter.Configuration if err := utils.LoadAndParse(asserterConfigurationFile, &asserterConfiguration); err != nil { - return nil, nil, err + return fmt.Errorf("%w: failure loading / parsing asserter-configuration-file", err) } resp, fetchErr := f.NetworkOptions(ctx, network, nil) if fetchErr != nil { - return nil, nil, fetchErr.Err + return fmt.Errorf("%w: failure getting /network/options", fetchErr.Err) } - return resp.Allow, &asserterConfiguration, nil + return validateNetworkAndAsserterAllowMatch(resp.Allow, &asserterConfiguration) } func validateNetworkAndAsserterAllowMatch( networkAllow *types.Allow, asserterConfiguration *asserter.Configuration, ) error { if networkAllow == nil { - return errors.New("networkAllow nil") + return errors.New("/network/options object's Allow is nil") } if asserterConfiguration == nil { - return errors.New("asserterConfiguration nil") + return errors.New("asserter-configuration-file object is nil") } if err := verifyTimestampStartIndex( @@ -86,7 +85,7 @@ func verifyTimestampStartIndex(networkTsi *int64, assertTsi int64) error { } if *networkTsi != assertTsi { return fmt.Errorf( - "network and asserter timestamp start index mismatch. %d %d", + "/network/options / asserter-configuration-file timestamp start index mismatch. %d %d", *networkTsi, assertTsi, ) } @@ -97,7 +96,7 @@ func verifyTimestampStartIndex(networkTsi *int64, assertTsi int64) error { func verifyOperationTypes(networkOt, asserterOt []string) error { if len(networkOt) != len(asserterOt) { return fmt.Errorf( - "network and asserter have operation type list size mismatch %v %v", + "/network/options / asserter-configuration-file operation types length mismatch %v %v", networkOt, asserterOt, ) } @@ -109,7 +108,7 @@ func verifyOperationTypes(networkOt, asserterOt []string) error { asserterOperationType := asserterOt[i] if networkOperationType != asserterOperationType { return fmt.Errorf( - "network / asserter operation type mismatch %v %v", + "/network/options / asserter-configuration-file operation type mismatch %v %v", networkOperationType, asserterOperationType, ) } @@ -121,7 +120,8 @@ func verifyOperationTypes(networkOt, asserterOt []string) error { func verifyOperationStatuses(networkOs, asserterOs []*types.OperationStatus) error { if len(networkOs) != len(asserterOs) { return fmt.Errorf( - "network and asserter have operation status list size mismatch %v %v", + "/network/options / asserter-configuration-file operation statuses length mismatch "+ + "%v %v", networkOs, asserterOs, ) } @@ -137,7 +137,7 @@ func verifyOperationStatuses(networkOs, asserterOs []*types.OperationStatus) err asserterOperationStatus := asserterOs[i] if !reflect.DeepEqual(networkOperationStatus, asserterOperationStatus) { return fmt.Errorf( - "network / asserter operation status mismatch %v %v", + "/network/options / asserter-configuration-file operation status mismatch %v %v", networkOperationStatus, asserterOperationStatus, ) } @@ -149,7 +149,7 @@ func verifyOperationStatuses(networkOs, asserterOs []*types.OperationStatus) err func verifyErrors(networkErrors, asserterErrors []*types.Error) error { if len(networkErrors) != len(asserterErrors) { return fmt.Errorf( - "network and asserter have error list size mismatch %v %v", + "/network/options / asserter-configuration-file errors length mismatch %v %v", networkErrors, asserterErrors, ) } @@ -165,7 +165,7 @@ func verifyErrors(networkErrors, asserterErrors []*types.Error) error { asserterError := asserterErrors[i] if !reflect.DeepEqual(networkError, asserterError) { return fmt.Errorf( - "network / asserter error mismatch %v %v", + "/network/options / asserter-configuration-file error mismatch %v %v", networkError, asserterError, ) } From c4d6a029273781ab3bb49ba3003479801b7ac9c6 Mon Sep 17 00:00:00 2001 From: Raghav Sharma Date: Wed, 17 Mar 2021 13:26:03 -0700 Subject: [PATCH 265/376] Better error messages --- cmd/validate_asserter_config.go | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/cmd/validate_asserter_config.go b/cmd/validate_asserter_config.go index 5e3be606..e37303bf 100644 --- a/cmd/validate_asserter_config.go +++ b/cmd/validate_asserter_config.go @@ -108,8 +108,9 @@ func verifyOperationTypes(networkOt, asserterOt []string) error { asserterOperationType := asserterOt[i] if networkOperationType != asserterOperationType { return fmt.Errorf( - "/network/options / asserter-configuration-file operation type mismatch %v %v", - networkOperationType, asserterOperationType, + "/network/options / asserter-configuration-file operation type mismatch %v %v\n"+ + "network operation types: %v\nasserter operation types: %v", + networkOperationType, asserterOperationType, networkOt, asserterOt, ) } } @@ -137,8 +138,9 @@ func verifyOperationStatuses(networkOs, asserterOs []*types.OperationStatus) err asserterOperationStatus := asserterOs[i] if !reflect.DeepEqual(networkOperationStatus, asserterOperationStatus) { return fmt.Errorf( - "/network/options / asserter-configuration-file operation status mismatch %v %v", - networkOperationStatus, asserterOperationStatus, + "/network/options / asserter-configuration-file operation status mismatch %v %v\n"+ + "network operation statuses: %v\nasserter operation statuses: %v", + networkOperationStatus, asserterOperationStatus, networkOs, asserterOs, ) } } @@ -165,8 +167,9 @@ func verifyErrors(networkErrors, asserterErrors []*types.Error) error { asserterError := asserterErrors[i] if !reflect.DeepEqual(networkError, asserterError) { return fmt.Errorf( - "/network/options / asserter-configuration-file error mismatch %v %v", - networkError, asserterError, + "/network/options / asserter-configuration-file error mismatch %v %v\n"+ + "network errors: %v\nasserter errors: %v", + networkError, asserterError, networkErrors, asserterErrors, ) } } From 13b38684d32de2a3cf5e1990eb62baf71b693f45 Mon Sep 17 00:00:00 2001 From: Xiaying Peng Date: Thu, 18 Mar 2021 14:45:50 -0700 Subject: [PATCH 266/376] log txn to terminal --- pkg/logger/logger.go | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/pkg/logger/logger.go b/pkg/logger/logger.go index e5ef440d..0097a9cf 100644 --- a/pkg/logger/logger.go +++ b/pkg/logger/logger.go @@ -270,12 +270,16 @@ func (l *Logger) TransactionStream( defer closeFile(f) for _, tx := range block.Transactions { - _, err = f.WriteString(fmt.Sprintf( + blockString := fmt.Sprintf( "Transaction %s at Block %d:%s\n", tx.TransactionIdentifier.Hash, block.BlockIdentifier.Index, block.BlockIdentifier.Hash, - )) + ) + + fmt.Print(blockString) + _, err = f.WriteString(blockString) + if err != nil { return err } From d3d6880c3801d2d1e5e746a7ccc95304cc47a708 Mon Sep 17 00:00:00 2001 From: Xiaying Peng Date: Fri, 19 Mar 2021 10:55:03 -0700 Subject: [PATCH 267/376] rename var name --- pkg/logger/logger.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pkg/logger/logger.go b/pkg/logger/logger.go index 0097a9cf..91b16ac8 100644 --- a/pkg/logger/logger.go +++ b/pkg/logger/logger.go @@ -270,15 +270,15 @@ func (l *Logger) TransactionStream( defer closeFile(f) for _, tx := range block.Transactions { - blockString := fmt.Sprintf( + transactionString := fmt.Sprintf( "Transaction %s at Block %d:%s\n", tx.TransactionIdentifier.Hash, block.BlockIdentifier.Index, block.BlockIdentifier.Hash, ) - fmt.Print(blockString) - _, err = f.WriteString(blockString) + fmt.Print(transactionString) + _, err = f.WriteString(transactionString) if err != nil { return err From 6a70d534d844d0350c7e5725a22c8536f9daeb4c Mon Sep 17 00:00:00 2001 From: Raghav Sharma Date: Mon, 22 Mar 2021 17:54:59 -0700 Subject: [PATCH 268/376] %+v --- cmd/validate_asserter_config.go | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/cmd/validate_asserter_config.go b/cmd/validate_asserter_config.go index e37303bf..03488041 100644 --- a/cmd/validate_asserter_config.go +++ b/cmd/validate_asserter_config.go @@ -96,7 +96,8 @@ func verifyTimestampStartIndex(networkTsi *int64, assertTsi int64) error { func verifyOperationTypes(networkOt, asserterOt []string) error { if len(networkOt) != len(asserterOt) { return fmt.Errorf( - "/network/options / asserter-configuration-file operation types length mismatch %v %v", + "/network/options / asserter-configuration-file operation types length mismatch %+v "+ + "%+v", networkOt, asserterOt, ) } @@ -108,8 +109,8 @@ func verifyOperationTypes(networkOt, asserterOt []string) error { asserterOperationType := asserterOt[i] if networkOperationType != asserterOperationType { return fmt.Errorf( - "/network/options / asserter-configuration-file operation type mismatch %v %v\n"+ - "network operation types: %v\nasserter operation types: %v", + "/network/options / asserter-configuration-file operation type mismatch %+v "+ + "%+v\nnetwork operation types: %+v\nasserter operation types: %+v", networkOperationType, asserterOperationType, networkOt, asserterOt, ) } @@ -122,7 +123,7 @@ func verifyOperationStatuses(networkOs, asserterOs []*types.OperationStatus) err if len(networkOs) != len(asserterOs) { return fmt.Errorf( "/network/options / asserter-configuration-file operation statuses length mismatch "+ - "%v %v", + "%+v %+v", networkOs, asserterOs, ) } @@ -138,8 +139,8 @@ func verifyOperationStatuses(networkOs, asserterOs []*types.OperationStatus) err asserterOperationStatus := asserterOs[i] if !reflect.DeepEqual(networkOperationStatus, asserterOperationStatus) { return fmt.Errorf( - "/network/options / asserter-configuration-file operation status mismatch %v %v\n"+ - "network operation statuses: %v\nasserter operation statuses: %v", + "/network/options / asserter-configuration-file operation status mismatch %+v "+ + "%+v\nnetwork operation statuses: %+v\nasserter operation statuses: %+v", networkOperationStatus, asserterOperationStatus, networkOs, asserterOs, ) } @@ -151,7 +152,7 @@ func verifyOperationStatuses(networkOs, asserterOs []*types.OperationStatus) err func verifyErrors(networkErrors, asserterErrors []*types.Error) error { if len(networkErrors) != len(asserterErrors) { return fmt.Errorf( - "/network/options / asserter-configuration-file errors length mismatch %v %v", + "/network/options / asserter-configuration-file errors length mismatch %+v %+v", networkErrors, asserterErrors, ) } @@ -167,8 +168,8 @@ func verifyErrors(networkErrors, asserterErrors []*types.Error) error { asserterError := asserterErrors[i] if !reflect.DeepEqual(networkError, asserterError) { return fmt.Errorf( - "/network/options / asserter-configuration-file error mismatch %v %v\n"+ - "network errors: %v\nasserter errors: %v", + "/network/options / asserter-configuration-file error mismatch %+v %+v\n"+ + "network errors: %+v\nasserter errors: %+v", networkError, asserterError, networkErrors, asserterErrors, ) } From f4b4a9f7889e3ee8abf1e14bb55e749078a94e83 Mon Sep 17 00:00:00 2001 From: Raghav Sharma Date: Tue, 30 Mar 2021 14:39:19 -0700 Subject: [PATCH 269/376] default to 1 --- cmd/validate_asserter_config.go | 9 +++++---- cmd/validate_asserter_config_test.go | 3 +++ 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/cmd/validate_asserter_config.go b/cmd/validate_asserter_config.go index 03488041..068deba7 100644 --- a/cmd/validate_asserter_config.go +++ b/cmd/validate_asserter_config.go @@ -80,13 +80,14 @@ func validateNetworkAndAsserterAllowMatch( } func verifyTimestampStartIndex(networkTsi *int64, assertTsi int64) error { - if networkTsi == nil { // This field is optional and defaults to all allowed - return nil + var networkTsiVal int64 = 1 + if networkTsi != nil { // This field is optional and defaults to all allowed + networkTsiVal = *networkTsi } - if *networkTsi != assertTsi { + if networkTsiVal != assertTsi { return fmt.Errorf( "/network/options / asserter-configuration-file timestamp start index mismatch. %d %d", - *networkTsi, assertTsi, + networkTsiVal, assertTsi, ) } diff --git a/cmd/validate_asserter_config_test.go b/cmd/validate_asserter_config_test.go index 12d2c24f..e48bf9c8 100644 --- a/cmd/validate_asserter_config_test.go +++ b/cmd/validate_asserter_config_test.go @@ -33,8 +33,11 @@ func TestNil(t *testing.T) { } func TestTsi(t *testing.T) { + // Confirm nil defaults to 1 networkAllow, asserterConfiguration := generateNetworkAllowAndAsserterConfiguration() networkAllow.TimestampStartIndex = nil + confirmError(t, networkAllow, asserterConfiguration) + asserterConfiguration.AllowedTimestampStartIndex = 1 confirmSuccess(t, networkAllow, asserterConfiguration) networkAllow, asserterConfiguration = generateNetworkAllowAndAsserterConfiguration() From 57642a422d8fd497153938aa09a996e03e4b2870 Mon Sep 17 00:00:00 2001 From: Nathan Smith Date: Mon, 29 Mar 2021 13:05:23 -0600 Subject: [PATCH 270/376] Recommend go replace when developing rosetta-cli --- README.md | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 2d957dee..20bb7f5e 100644 --- a/README.md +++ b/README.md @@ -568,6 +568,11 @@ operations should be returned by the Rosetta Data API. * `make release` to run one last check before opening a PR * `make compile version=RELEASE_TAG` to generate binaries +If you are developing on both rosetta-cli and rosetta-sdk-go, use [go.mod replace](https://golang.org/ref/mod#go-mod-file-replace) to reference local changes: +``` +replace "github.com/coinbase/rosetta-sdk-go" v0.6.8 => "../rosetta-sdk-go" +``` + ### Helper/Handler Many of the packages use a `Helper/Handler` interface pattern to acquire required information or to send events to some client implementation. An example @@ -597,4 +602,4 @@ pkg ## License This project is available open source under the terms of the [Apache 2.0 License](https://opensource.org/licenses/Apache-2.0). -© 2020 Coinbase +© 2021 Coinbase From 392f436beb6f9d3226a53d10481f86bf11f2876c Mon Sep 17 00:00:00 2001 From: Madhur Shrimal Date: Fri, 27 Aug 2021 14:51:53 -0700 Subject: [PATCH 271/376] update scripts to follow go 1.16 (#237) * update scripts to follow go 1.16 * update coveralls --- .circleci/config.yml | 2 +- Makefile | 15 +++++++++++---- go.mod | 2 +- 3 files changed, 13 insertions(+), 6 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 6db4d75e..12464874 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -16,7 +16,7 @@ version: 2.1 executors: default: docker: - - image: circleci/golang:1.15 + - image: circleci/golang:1.16 user: root # go directory is owned by root working_directory: /go/src/github.com/coinbase/rosetta-cli environment: diff --git a/Makefile b/Makefile index 644b0718..d77c7cd1 100644 --- a/Makefile +++ b/Makefile @@ -7,10 +7,13 @@ # it is necessary to use `go run `. Running `go get` does # not install any binaries that could be used to run # the commands directly. -ADDLICENSE_CMD=go run github.com/google/addlicense +ADDLICENSE_INSTALL=go install github.com/google/addlicense@latest +ADDLICENSE_CMD=addlicense ADDLICENCE_SCRIPT=${ADDLICENSE_CMD} -c "Coinbase, Inc." -l "apache" -v -GOLINES_CMD=go run github.com/segmentio/golines -GOVERALLS_CMD=go run github.com/mattn/goveralls +GOLINES_INSTALL=go install github.com/segmentio/golines@latest +GOLINES_CMD=golines +GOVERALLS_INSTALL=go install github.com/mattn/goveralls@latest +GOVERALLS_CMD=goveralls COVERAGE_TEST_DIRECTORIES=./configuration/... ./pkg/results/... \ ./pkg/logger/... ./cmd TEST_SCRIPT=go test -v ./pkg/... ./configuration/... ./cmd @@ -38,16 +41,20 @@ validate-configuration-files: test: | validate-configuration-files ${TEST_SCRIPT} -test-cover: +test-cover: + ${GOVERALLS_INSTALL} if [ "${COVERALLS_TOKEN}" ]; then ${COVERAGE_TEST_SCRIPT} -coverprofile=c.out -covermode=count; ${GOVERALLS_CMD} -coverprofile=c.out -repotoken ${COVERALLS_TOKEN}; fi add-license: + ${ADDLICENSE_INSTALL} ${ADDLICENCE_SCRIPT} . check-license: + ${ADDLICENSE_INSTALL} ${ADDLICENCE_SCRIPT} -check . shorten-lines: + ${GOLINES_INSTALL} ${GOLINES_CMD} -w --shorten-comments pkg cmd configuration salus: diff --git a/go.mod b/go.mod index 73c5a30b..54e14d7a 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/coinbase/rosetta-cli -go 1.13 +go 1.16 require ( github.com/coinbase/rosetta-sdk-go v0.6.8 From 73be3b794de85416f436415427c7b50a963c3ce6 Mon Sep 17 00:00:00 2001 From: Madhur Shrimal Date: Wed, 1 Sep 2021 11:19:44 -0700 Subject: [PATCH 272/376] Upgrade to rosetta-sdk@0.7.0 (#236) * upgrade to rosetta-sdk 0.7.0 --- cmd/check_construction.go | 2 +- cmd/check_data.go | 2 +- cmd/utils_asserter_configuration.go | 2 +- cmd/view_balance.go | 2 +- cmd/view_block.go | 2 +- configuration/configuration.go | 9 +++++++ configuration/types.go | 5 ++++ go.mod | 2 +- go.sum | 37 ++--------------------------- pkg/results/data_results_test.go | 3 +++ 10 files changed, 25 insertions(+), 41 deletions(-) diff --git a/cmd/check_construction.go b/cmd/check_construction.go index 4a0febf7..d4d24e26 100644 --- a/cmd/check_construction.go +++ b/cmd/check_construction.go @@ -81,7 +81,7 @@ func runCheckConstructionCmd(_ *cobra.Command, _ []string) error { fetcherOpts..., ) - _, _, fetchErr := fetcher.InitializeAsserter(ctx, Config.Network) + _, _, fetchErr := fetcher.InitializeAsserter(ctx, Config.Network, Config.ValidationFile) if fetchErr != nil { cancel() return results.ExitConstruction( diff --git a/cmd/check_data.go b/cmd/check_data.go index 46eb5dbc..27c8bcb0 100644 --- a/cmd/check_data.go +++ b/cmd/check_data.go @@ -89,7 +89,7 @@ func runCheckDataCmd(_ *cobra.Command, _ []string) error { fetcherOpts..., ) - _, _, fetchErr := fetcher.InitializeAsserter(ctx, Config.Network) + _, _, fetchErr := fetcher.InitializeAsserter(ctx, Config.Network, "") if fetchErr != nil { cancel() return results.ExitData( diff --git a/cmd/utils_asserter_configuration.go b/cmd/utils_asserter_configuration.go index ff70365f..3f4b4e83 100644 --- a/cmd/utils_asserter_configuration.go +++ b/cmd/utils_asserter_configuration.go @@ -53,7 +53,7 @@ func runCreateConfigurationCmd(cmd *cobra.Command, args []string) error { ) // Initialize the fetcher's asserter - _, _, fetchErr := newFetcher.InitializeAsserter(Context, Config.Network) + _, _, fetchErr := newFetcher.InitializeAsserter(Context, Config.Network, Config.ValidationFile) if fetchErr != nil { return fmt.Errorf("%w: failed to initialize asserter", fetchErr.Err) } diff --git a/cmd/view_balance.go b/cmd/view_balance.go index 32ca3da1..3562bd38 100644 --- a/cmd/view_balance.go +++ b/cmd/view_balance.go @@ -72,7 +72,7 @@ func runViewBalanceCmd(cmd *cobra.Command, args []string) error { ) // Initialize the fetcher's asserter - _, _, fetchErr := newFetcher.InitializeAsserter(Context, Config.Network) + _, _, fetchErr := newFetcher.InitializeAsserter(Context, Config.Network, "") if fetchErr != nil { return fmt.Errorf("%w: unable to initialize asserter", fetchErr.Err) } diff --git a/cmd/view_block.go b/cmd/view_block.go index 3de96074..25031f3d 100644 --- a/cmd/view_block.go +++ b/cmd/view_block.go @@ -94,7 +94,7 @@ func runViewBlockCmd(_ *cobra.Command, args []string) error { // Behind the scenes this makes a call to get the // network status and uses the response to inform // the asserter what are valid responses. - _, _, fetchErr := newFetcher.InitializeAsserter(Context, Config.Network) + _, _, fetchErr := newFetcher.InitializeAsserter(Context, Config.Network, "") if fetchErr != nil { return fmt.Errorf("%w: unable to initialize asserter", fetchErr.Err) } diff --git a/configuration/configuration.go b/configuration/configuration.go index 2d9847f5..1dc1a2f9 100644 --- a/configuration/configuration.go +++ b/configuration/configuration.go @@ -22,6 +22,7 @@ import ( "log" "path" "runtime" + "strings" "github.com/coinbase/rosetta-sdk-go/asserter" "github.com/coinbase/rosetta-sdk-go/constructor/dsl" @@ -163,6 +164,10 @@ func populateMissingFields(config *Configuration) *Configuration { config.SerialBlockWorkers = numCPU } + if len(strings.TrimSpace(config.ValidationFile)) == 0 { + config.ValidationFile = "" + } + config.Construction = populateConstructionMissingFields(config.Construction) config.Data = populateDataMissingFields(config.Data) @@ -348,6 +353,10 @@ func modifyFilePaths(config *Configuration, fileDir string) { ) } } + + if len(config.ValidationFile) > 0 { + config.ValidationFile = path.Join(fileDir, config.ValidationFile) + } } // LoadConfiguration returns a parsed and asserted Configuration for running diff --git a/configuration/types.go b/configuration/types.go index 13ee43e6..817608aa 100644 --- a/configuration/types.go +++ b/configuration/types.go @@ -416,6 +416,11 @@ type Configuration struct { // If not populated, this value defaults to runtime.NumCPU(). SerialBlockWorkers int `json:"serial_block_workers,omitempty"` + // ValidationFile is the file used for asset specific validation + // If not provided, then this will be empty string and no asset + // specific validation will be done + ValidationFile string `json:"validation_file,omitempty"` + Construction *ConstructionConfiguration `json:"construction"` Data *DataConfiguration `json:"data"` } diff --git a/go.mod b/go.mod index 54e14d7a..6864de17 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/coinbase/rosetta-cli go 1.16 require ( - github.com/coinbase/rosetta-sdk-go v0.6.8 + github.com/coinbase/rosetta-sdk-go v0.7.0 github.com/fatih/color v1.10.0 github.com/olekukonko/tablewriter v0.0.2-0.20190409134802-7e037d187b0c github.com/spf13/cobra v1.1.1 diff --git a/go.sum b/go.sum index 585aabe9..b447a01b 100644 --- a/go.sum +++ b/go.sum @@ -26,15 +26,12 @@ github.com/Azure/go-autorest/logger v0.1.0/go.mod h1:oExouG+K6PryycPJfVSxi/koC6L github.com/Azure/go-autorest/tracing v0.5.0/go.mod h1:r/s2XiOKccPW3HrqB+W0TQzfbtp2fGCgRFtBroKn4Dk= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= -github.com/DataDog/zstd v1.4.1 h1:3oxKN3wbHibqx897utPC2LTQU4J+IHWWJO+glkAkpFM= github.com/DataDog/zstd v1.4.1/go.mod h1:1jcaCB/ufaK+sKp1NBhlGmpz41jOoPQ35bpF36t7BBo= github.com/DataDog/zstd v1.4.5 h1:EndNeuB0l9syBZhut0wns3gV1hL8zX8LIu6ZiVHWLIQ= github.com/DataDog/zstd v1.4.5/go.mod h1:1jcaCB/ufaK+sKp1NBhlGmpz41jOoPQ35bpF36t7BBo= github.com/OneOfOne/xxhash v1.2.2 h1:KMrpdQIwFcEqXDklaen+P1axHaj9BSKzvpUUfnHldSE= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= -github.com/StackExchange/wmi v0.0.0-20180116203802-5d049714c4a6 h1:fLjPD/aNc3UIOA6tDi6QXUemppXK3P9BI7mr2hd6gx8= github.com/StackExchange/wmi v0.0.0-20180116203802-5d049714c4a6/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg= -github.com/VictoriaMetrics/fastcache v1.5.7 h1:4y6y0G8PRzszQUYIQHHssv/jgPHAb5qQuuDNdCbyAgw= github.com/VictoriaMetrics/fastcache v1.5.7/go.mod h1:ptDBkNMQI4RtmVo8VS/XwRY6RoTu1dAWCbrk+6WsEM8= github.com/Zilliqa/gozilliqa-sdk v1.2.1-0.20201201074141-dd0ecada1be6 h1:1d9pzdbkth4D9AX6ndKSl7of3UTV0RYl3z64U2dXMGo= github.com/Zilliqa/gozilliqa-sdk v1.2.1-0.20201201074141-dd0ecada1be6/go.mod h1:eSYp2T6f0apnuW8TzhV3f6Aff2SE8Dwio++U4ha4yEM= @@ -42,7 +39,6 @@ github.com/aead/siphash v1.0.1/go.mod h1:Nywa3cDsYNNK3gaciGTWPwHt0wlpNV15vwmswBA github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/allegro/bigcache v1.2.1-0.20190218064605-e24eb225f156/go.mod h1:Cb/ax3seSYIx7SuZdm2G2xzfwmv3TPSk2ucNfQESPXM= -github.com/aristanetworks/goarista v0.0.0-20170210015632-ea17b1a17847 h1:rtI0fD4oG/8eVokGVPYJEW1F88p1ZNgXiEIs9thEE4A= github.com/aristanetworks/goarista v0.0.0-20170210015632-ea17b1a17847/go.mod h1:D/tb0zPVXnP7fmsLZjtdUhSsumbK/ij54UXjjVgMGxQ= github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= @@ -55,7 +51,6 @@ github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kB github.com/bketelsen/crypt v0.0.3-0.20200106085610-5cbc8cc4026c/go.mod h1:MKsuJmJgSg28kpZDP6UIiPt0e0Oz0kqKNGyRaWEPv84= github.com/btcsuite/btcd v0.0.0-20171128150713-2e60448ffcc6/go.mod h1:Dmm/EzmjnCiweXmzRIAiUWCInVmPgjkzgv5k4tVyXiQ= github.com/btcsuite/btcd v0.0.0-20190315201642-aa6e0f35703c/go.mod h1:DrZx5ec/dmnfpw9KyYoQyYo7d0KEvTkk/5M/vbZjAr8= -github.com/btcsuite/btcd v0.20.1-beta h1:Ik4hyJqN8Jfyv3S4AGBOmyouMsYE3EdYODkMbQjwPGw= github.com/btcsuite/btcd v0.20.1-beta/go.mod h1:wVuoA8VJLEcwgqHBwHmzLRazpKxTv13Px/pDuV7OomQ= github.com/btcsuite/btcd v0.21.0-beta h1:At9hIZdJW0s9E/fAz28nrz6AmcNlSVucCH796ZteX1M= github.com/btcsuite/btcd v0.21.0-beta/go.mod h1:ZSWyehm27aAuS9bvkATT+Xte3hjHZ+MRgMY/8NJ7K94= @@ -76,12 +71,11 @@ github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA github.com/cespare/cp v0.1.0/go.mod h1:SOGHArjBr4JWaSDEVpWpo/hNg6RoKrls6Oh40hiwW+s= github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= -github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+qY= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cloudflare/cloudflare-go v0.10.2-0.20190916151808-a80f83b9add9/go.mod h1:1MxXX1Ux4x6mqPmjkUgTP1CdXIBXKX7T+Jk9Gxrmx+U= -github.com/coinbase/rosetta-sdk-go v0.6.8 h1:DCWvfpzN9HkKux0eBZwN+9b7vstP4ij77zpyHtYy8dk= -github.com/coinbase/rosetta-sdk-go v0.6.8/go.mod h1:jczGuxueJURzr7J8/OeZM3FRbezjw8lWIMxXOBLONOM= +github.com/coinbase/rosetta-sdk-go v0.7.0 h1:lmTO/JEpCvZgpbkOITL95rA80CPKb5CtMzLaqF2mCNg= +github.com/coinbase/rosetta-sdk-go v0.7.0/go.mod h1:7nD3oBPIiHqhRprqvMgPoGxe/nyq3yftRmpsy29coWE= github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= @@ -105,7 +99,6 @@ github.com/dgraph-io/ristretto v0.0.3-0.20200630154024-f66de99634de/go.mod h1:KP github.com/dgraph-io/ristretto v0.0.3 h1:jh22xisGBjrEVnRZ1DVTpBVQm0Xndu8sMl0CWDzSIBI= github.com/dgraph-io/ristretto v0.0.3/go.mod h1:KPxhHT9ZxKefz+PCeOGsrHpl1qZ7i70dGTu2u+Ahh6E= github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= -github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2 h1:tdlZCpZ/P9DhczCTSixgIKmwPv6+wP5DGjqLYw5SUiA= github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 h1:fAjc9m62+UWV/WAFKLNi6ZS0675eEUC9y3AlwSbQu1Y= github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= @@ -134,10 +127,8 @@ github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9 github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= -github.com/go-ole/go-ole v1.2.1 h1:2lOsA72HgjxAuMlKpFiCbHTvu44PIVkZ5hqm3RSdI/E= github.com/go-ole/go-ole v1.2.1/go.mod h1:7FAglXiTm7HKlQRDeOQ6ZNUHidzCWXuZWq/1dTyBNF8= github.com/go-sourcemap/sourcemap v2.1.2+incompatible/go.mod h1:F8jJfvm2KbVjc5NqelyYJmf/v5J0dwNLS2mL4sNA1Jg= -github.com/go-stack/stack v1.8.0 h1:5SgMzNM5HxrEjV0ww2lTmX6E2Izsfxas4+YHWRs3Lsk= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= @@ -147,7 +138,6 @@ github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfb github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.3.1 h1:YF8+flBXS5eO826T4nzqPrxfhQThhXl0YzfuUPu4SBg= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= @@ -166,9 +156,7 @@ github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= -github.com/google/go-cmp v0.4.0 h1:xsAVV57WRhGj6kEIi8ReJzQlHHqcBYCElAvkovg3B/4= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.0 h1:/QaMHBdZ26BB3SSst0Iwl10Epc+xhTquomWX0oZEB6w= github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.2 h1:X2ev0eStA3AbceY54o37/0PQ/UWqKEiiO2dKL5OPaFM= github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= @@ -297,7 +285,6 @@ github.com/pborman/uuid v0.0.0-20170112150404-1b00554d8222/go.mod h1:VyrYX9gd7ir github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= github.com/peterh/liner v1.1.1-0.20190123174540-a2c9a5303de7/go.mod h1:CRroGNssyjTd/qIG2FyxByd2S8JEAZXBl4qUrZf8GS0= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= @@ -326,7 +313,6 @@ github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= github.com/segmentio/fasthash v1.0.3 h1:EI9+KE1EwvMLBWwjpRDc+fEM+prwxDYbslddQGtrmhM= github.com/segmentio/fasthash v1.0.3/go.mod h1:waKX8l2N8yckOgmSsXJi7x1ZfdKZ4x7KRMzBtS3oedY= -github.com/shirou/gopsutil v2.20.5+incompatible h1:tYH07UPoQt0OCQdgWWMgYHy3/a9bcxNpBIysykNIP7I= github.com/shirou/gopsutil v2.20.5+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA= github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= @@ -338,23 +324,18 @@ github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0b github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= -github.com/spf13/cobra v0.0.5 h1:f0B+LkLX6DtmRH1isoNA9VTtNUK9K8xYd28JNNfOv/s= github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU= github.com/spf13/cobra v1.1.1 h1:KfztREH0tPxJJ+geloSLaAkaPkr4ki2Er5quFV1TDo4= github.com/spf13/cobra v1.1.1/go.mod h1:WnodtKOvamDL/PwE2M4iKs8aMDBZ5Q5klgD3qfVJQMI= github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= -github.com/spf13/pflag v1.0.3 h1:zPAT6CGy6wXeQ7NtTnaTerfKOsV6V6F8agHXFiazDkg= github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s= github.com/spf13/viper v1.7.0/go.mod h1:8WkrPz2fc9jxqZNCJI/76HCieCp4Q8HaLFoCha5qpdg= github.com/status-im/keycard-go v0.0.0-20190316090335-8537d3370df4/go.mod h1:RZLeN1LMWmRsyYjvAu+I6Dm9QmlDaIIt+Y+4Kd7Tp+Q= -github.com/steakknife/bloomfilter v0.0.0-20180922174646-6819c0d2a570 h1:gIlAHnH1vJb5vwEjIp5kBj/eu99p/bl0Ay2goiPe5xE= github.com/steakknife/bloomfilter v0.0.0-20180922174646-6819c0d2a570/go.mod h1:8OR4w3TdeIHIh1g6EMY5p0gVNOovcWC+1vpc7naMuAw= -github.com/steakknife/hamming v0.0.0-20180906055917-c99c65617cd3 h1:njlZPzLwU639dk2kqnCPPv+wNjq7Xb6EfUxe/oX0/NM= github.com/steakknife/hamming v0.0.0-20180906055917-c99c65617cd3/go.mod h1:hpGUWaI9xL8pRQCTXQgocU38Qw1g0Us7n5PxxTwTCYU= -github.com/stretchr/objx v0.1.0 h1:4G4v2dO3VZwixGIRoQ5Lfboy6nUhCyYzaqnIAPPhYs4= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1 h1:2vfRuCMp5sSVIDSqO8oNnWJq7mPa6KVP3iPIwFBuy8A= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= @@ -362,7 +343,6 @@ github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXf github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= -github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= @@ -404,11 +384,9 @@ golang.org/x/crypto v0.0.0-20190426145343-a29dc8fdc734/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190909091759-094676da4a83/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550 h1:ObdrDkeb4kJdCP557AjRjq69pTHfNouLtWZG7j9rPN8= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200115085410-6d4e4cb37c7d/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200510223506-06a226fb4e37/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 h1:psW17arqaxU48Z5kZ0CQnkZWQJsqcURM6tKiBApRjXI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200820211705-5c72a883971a h1:vclmkQCjlDX5OydZ9wv8rBCcS0QyQY66Mpf/7BZbInM= golang.org/x/crypto v0.0.0-20200820211705-5c72a883971a/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= @@ -423,7 +401,6 @@ golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+o golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= -golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3 h1:XQyxROzUlZH+WIQwySDgnISgOivlhjIEwaQaJEJrrN0= golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= @@ -451,10 +428,8 @@ golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= -golang.org/x/net v0.0.0-20190620200207-3b0461eec859 h1:R/3boaszxrf1GEUWTVDzSKVwLmSJpwZ1yqXm8j0v2QI= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200813134508-3edf25e44fcc h1:zK/HqS5bZxDptfPJNq8v7vJfXtkU7r9TLIoSr1bXaP4= golang.org/x/net v0.0.0-20200813134508-3edf25e44fcc/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20200904194848-62affa334b73 h1:MXfv8rhZWmFeqX3GNZRsd6vOLoaCHjYEX3qkRo3YBUA= @@ -484,7 +459,6 @@ golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb h1:fgwFCsaw9buMuxNd6+DQfAuSFqbNiQZpcgJQAgJsK6k= golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -522,11 +496,9 @@ golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtn golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191112195655-aa38f8e97acc/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20200117012304-6edc0a871e69 h1:yBHHx+XZqXJBm6Exke3N7V9gnlsyXxoCPEb1yVenjfk= golang.org/x/tools v0.0.0-20200117012304-6edc0a871e69/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -560,16 +532,12 @@ google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQ google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= -google.golang.org/protobuf v1.23.0 h1:4MY060fB1DLGMB/7MBTLnwQUY6+F09GEiz6SsrNqyzM= google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.25.0 h1:Ejskq+SyPohKW+1uil0JJMtmHCgJPJ/qWTxr8qp+R4c= google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= -gopkg.in/alecthomas/kingpin.v2 v2.2.6 h1:jMFz6MfLP0/4fUyZle81rXUoxOBFi19VUFKVDOQfozc= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= -gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= @@ -583,7 +551,6 @@ gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWD gopkg.in/urfave/cli.v1 v1.20.0/go.mod h1:vuBzUtMdQeixQj8LVd+/98pzhxNGQoyuPBlsXHOQNO0= gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= diff --git a/pkg/results/data_results_test.go b/pkg/results/data_results_test.go index d5b92c1b..9d687817 100644 --- a/pkg/results/data_results_test.go +++ b/pkg/results/data_results_test.go @@ -57,6 +57,9 @@ func baseAsserter() *asserter.Asserter { }, []*types.Error{}, nil, + &asserter.Validations{ + Enabled: false, + }, ) return a } From 5c076d153ae1c54b89c886ffb20d37a5af513027 Mon Sep 17 00:00:00 2001 From: Madhur Shrimal Date: Wed, 8 Sep 2021 10:37:38 -0700 Subject: [PATCH 273/376] Use crazy-max/xgo instead of karalabke/xgo (#241) * update to crazy-max/xgo * add arm64 --- Makefile | 1 + scripts/compile.sh | 14 ++++++++++++-- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index d77c7cd1..5fdf9434 100644 --- a/Makefile +++ b/Makefile @@ -62,6 +62,7 @@ salus: release: add-license shorten-lines format test lint salus +# This command is to generate multi-platform binaries. compile: ./scripts/compile.sh $(version) diff --git a/scripts/compile.sh b/scripts/compile.sh index c5b0d50c..08d3872c 100755 --- a/scripts/compile.sh +++ b/scripts/compile.sh @@ -16,12 +16,22 @@ VERSION=$1; -xgo -go go-1.15.5 --targets=darwin/*,windows/*,linux/* -out "bin/rosetta-cli-${VERSION}" .; +go get github.com/crazy-max/xgo + +MAC_TARGETS="darwin/amd64,darwin/arm64" +LINUX_TARGETS="linux/amd64,linux/arm64,linux/mips64,linux/mips64le,linux/ppc64le,linux/s390x" +WINDOWS_TARGET="windows/amd64" +TARGETS="${MAC_TARGETS},${LINUX_TARGETS},${WINDOWS_TARGET}" + +xgo -go 1.16.3 --targets=${TARGETS} -out "bin/rosetta-cli-${VERSION}" .; # Rename some files -mv "bin/rosetta-cli-${VERSION}-darwin-10.6-amd64" "bin/rosetta-cli-${VERSION}-darwin-amd64" +mv "bin/rosetta-cli-${VERSION}-darwin-10.12-amd64" "bin/rosetta-cli-${VERSION}-darwin-amd64" +mv "bin/rosetta-cli-${VERSION}-darwin-10.12-amd64" "bin/rosetta-cli-${VERSION}-darwin-arm64" mv "bin/rosetta-cli-${VERSION}-windows-4.0-amd64.exe" "bin/rosetta-cli-${VERSION}-windows-amd64" # Tar all files cd bin || exit; for i in *; do tar -czf "$i.tar.gz" "$i" && rm "$i"; done + +go mod tidy From 5192235f578e2a5ecaa4d650a61d0a18dc6009e1 Mon Sep 17 00:00:00 2001 From: Madhur Shrimal Date: Wed, 8 Sep 2021 10:39:31 -0700 Subject: [PATCH 274/376] enable validation file for other commands (#239) --- cmd/check_data.go | 2 +- cmd/view_balance.go | 2 +- cmd/view_block.go | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/cmd/check_data.go b/cmd/check_data.go index 27c8bcb0..9e6d0e7e 100644 --- a/cmd/check_data.go +++ b/cmd/check_data.go @@ -89,7 +89,7 @@ func runCheckDataCmd(_ *cobra.Command, _ []string) error { fetcherOpts..., ) - _, _, fetchErr := fetcher.InitializeAsserter(ctx, Config.Network, "") + _, _, fetchErr := fetcher.InitializeAsserter(ctx, Config.Network, Config.ValidationFile) if fetchErr != nil { cancel() return results.ExitData( diff --git a/cmd/view_balance.go b/cmd/view_balance.go index 3562bd38..bfe3bbe2 100644 --- a/cmd/view_balance.go +++ b/cmd/view_balance.go @@ -72,7 +72,7 @@ func runViewBalanceCmd(cmd *cobra.Command, args []string) error { ) // Initialize the fetcher's asserter - _, _, fetchErr := newFetcher.InitializeAsserter(Context, Config.Network, "") + _, _, fetchErr := newFetcher.InitializeAsserter(Context, Config.Network, Config.ValidationFile) if fetchErr != nil { return fmt.Errorf("%w: unable to initialize asserter", fetchErr.Err) } diff --git a/cmd/view_block.go b/cmd/view_block.go index 25031f3d..597d34b5 100644 --- a/cmd/view_block.go +++ b/cmd/view_block.go @@ -94,7 +94,7 @@ func runViewBlockCmd(_ *cobra.Command, args []string) error { // Behind the scenes this makes a call to get the // network status and uses the response to inform // the asserter what are valid responses. - _, _, fetchErr := newFetcher.InitializeAsserter(Context, Config.Network, "") + _, _, fetchErr := newFetcher.InitializeAsserter(Context, Config.Network, Config.ValidationFile) if fetchErr != nil { return fmt.Errorf("%w: unable to initialize asserter", fetchErr.Err) } From a2a03d06ad39d621c9572377f735ccaaf05a626b Mon Sep 17 00:00:00 2001 From: Madhur Shrimal Date: Wed, 8 Sep 2021 11:01:40 -0700 Subject: [PATCH 275/376] quick fix script bug (#243) --- scripts/compile.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/compile.sh b/scripts/compile.sh index 08d3872c..40cb9f41 100755 --- a/scripts/compile.sh +++ b/scripts/compile.sh @@ -27,7 +27,7 @@ xgo -go 1.16.3 --targets=${TARGETS} -out "bin/rosetta-cli-${VERSION}" .; # Rename some files mv "bin/rosetta-cli-${VERSION}-darwin-10.12-amd64" "bin/rosetta-cli-${VERSION}-darwin-amd64" -mv "bin/rosetta-cli-${VERSION}-darwin-10.12-amd64" "bin/rosetta-cli-${VERSION}-darwin-arm64" +mv "bin/rosetta-cli-${VERSION}-darwin-10.12-arm64" "bin/rosetta-cli-${VERSION}-darwin-arm64" mv "bin/rosetta-cli-${VERSION}-windows-4.0-amd64.exe" "bin/rosetta-cli-${VERSION}-windows-amd64" # Tar all files From 769a0ba99d4911df4686cfa92b2875d77b00714c Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sun, 24 Oct 2021 16:11:40 -0700 Subject: [PATCH 276/376] Bump github.com/fatih/color from 1.10.0 to 1.13.0 (#244) Bumps [github.com/fatih/color](https://github.com/fatih/color) from 1.10.0 to 1.13.0. - [Release notes](https://github.com/fatih/color/releases) - [Commits](https://github.com/fatih/color/compare/v1.10.0...v1.13.0) --- updated-dependencies: - dependency-name: github.com/fatih/color dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 12 ++++++++---- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/go.mod b/go.mod index 6864de17..8b207148 100644 --- a/go.mod +++ b/go.mod @@ -4,7 +4,7 @@ go 1.16 require ( github.com/coinbase/rosetta-sdk-go v0.7.0 - github.com/fatih/color v1.10.0 + github.com/fatih/color v1.13.0 github.com/olekukonko/tablewriter v0.0.2-0.20190409134802-7e037d187b0c github.com/spf13/cobra v1.1.1 github.com/stretchr/testify v1.7.0 diff --git a/go.sum b/go.sum index b447a01b..1438fadd 100644 --- a/go.sum +++ b/go.sum @@ -116,8 +116,9 @@ github.com/ethereum/go-ethereum v1.9.25 h1:mMiw/zOOtCLdGLWfcekua0qPrJTe7FVIiHJ4I github.com/ethereum/go-ethereum v1.9.25/go.mod h1:vMkFiYLHI4tgPw4k2j4MHKoovchFE8plZ0M9VMk4/oM= github.com/fatih/color v1.3.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= -github.com/fatih/color v1.10.0 h1:s36xzo75JdqLaaWoiEHk767eHiwo0598uUxyfiPkDsg= github.com/fatih/color v1.10.0/go.mod h1:ELkj/draVOlAH/xkhN6mQ50Qd0MPOk5AAr3maGEBuJM= +github.com/fatih/color v1.13.0 h1:8LOYc1KYPPmyKMuN8QV2DNRWNbLo6LZ0iLs8+mlH53w= +github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk= github.com/fjl/memsize v0.0.0-20180418122429-ca190fb6ffbc/go.mod h1:VvhXpOYNQvB+uIk2RvXzuaQtkQJzzIx6lSBe1xv7hi0= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= @@ -236,14 +237,16 @@ github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czP github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= github.com/mattn/go-colorable v0.1.0/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= -github.com/mattn/go-colorable v0.1.8 h1:c1ghPdyEDarC70ftn0y+A/Ee++9zz8ljHG1b13eJ0s8= github.com/mattn/go-colorable v0.1.8/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= +github.com/mattn/go-colorable v0.1.9 h1:sqDoxXbdeALODt0DAeJCVp38ps9ZogZEAXjus69YV3U= +github.com/mattn/go-colorable v0.1.9/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= github.com/mattn/go-ieproxy v0.0.0-20190610004146-91bb50d98149/go.mod h1:31jz6HNzdxOmlERGGEc4v/dMssOfmp2p5bT/okiKFFc= github.com/mattn/go-ieproxy v0.0.0-20190702010315-6dee0af9227d/go.mod h1:31jz6HNzdxOmlERGGEc4v/dMssOfmp2p5bT/okiKFFc= github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-isatty v0.0.5-0.20180830101745-3fb116b82035/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= -github.com/mattn/go-isatty v0.0.12 h1:wuysRhFDzyxgEmMf5xjvJ2M9dZoWAXNNr5LSBS7uHXY= github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= +github.com/mattn/go-isatty v0.0.14 h1:yVuAays6BHfxijgZPzw+3Zlu5yQgKGP2/hcQbHb7S9Y= +github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= github.com/mattn/go-runewidth v0.0.3/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= github.com/mattn/go-runewidth v0.0.4 h1:2BvfKmzob6Bmd4YsL0zygOqfdFnK7GR4QL06Do4/p7Y= github.com/mattn/go-runewidth v0.0.4/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= @@ -469,8 +472,9 @@ golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200519105757-fe76b779f299/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200814200057-3d37ad5750ed/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200824131525-c12d262b63d8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200909081042-eff7692f9009 h1:W0lCpv29Hv0UaM1LXb9QlBHLNP8UFfcKjblhVCWftOM= golang.org/x/sys v0.0.0-20200909081042-eff7692f9009/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c h1:F1jZWGFhYfh0Ci55sIpILtKKK8p3i2/krTr0H1rg74I= +golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= From 470a87de1bf58054372dad535a7c90e6c903ea2b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 25 Oct 2021 11:31:03 -0700 Subject: [PATCH 277/376] Bump github.com/spf13/cobra from 1.1.1 to 1.2.1 (#233) Bumps [github.com/spf13/cobra](https://github.com/spf13/cobra) from 1.1.1 to 1.2.1. - [Release notes](https://github.com/spf13/cobra/releases) - [Changelog](https://github.com/spf13/cobra/blob/master/CHANGELOG.md) - [Commits](https://github.com/spf13/cobra/compare/v1.1.1...v1.2.1) --- updated-dependencies: - dependency-name: github.com/spf13/cobra dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 4 +- go.sum | 371 ++++++++++++++++++++++++++++++++++++++++++++++++--------- 2 files changed, 319 insertions(+), 56 deletions(-) diff --git a/go.mod b/go.mod index 8b207148..69381e90 100644 --- a/go.mod +++ b/go.mod @@ -6,7 +6,7 @@ require ( github.com/coinbase/rosetta-sdk-go v0.7.0 github.com/fatih/color v1.13.0 github.com/olekukonko/tablewriter v0.0.2-0.20190409134802-7e037d187b0c - github.com/spf13/cobra v1.1.1 + github.com/spf13/cobra v1.2.1 github.com/stretchr/testify v1.7.0 - golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208 + golang.org/x/sync v0.0.0-20210220032951-036812b2e83c ) diff --git a/go.sum b/go.sum index 1438fadd..21d064c1 100644 --- a/go.sum +++ b/go.sum @@ -5,11 +5,37 @@ cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6A cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc= cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0= +cloud.google.com/go v0.50.0/go.mod h1:r9sluTvynVuxRIOHXQEHMFffphuXHOMZMycpNR5e6To= +cloud.google.com/go v0.52.0/go.mod h1:pXajvRH/6o3+F9jDHZWQ5PbGhn+o8w9qiu/CffaVdO4= +cloud.google.com/go v0.53.0/go.mod h1:fp/UouUEsRkN6ryDKNW/Upv/JBKnv6WDthjR6+vze6M= +cloud.google.com/go v0.54.0/go.mod h1:1rq2OEkV3YMf6n/9ZvGWI3GWw0VoqH/1x2nd8Is/bPc= +cloud.google.com/go v0.56.0/go.mod h1:jr7tqZxxKOVYizybht9+26Z/gUq7tiRzu+ACVAMbKVk= +cloud.google.com/go v0.57.0/go.mod h1:oXiQ6Rzq3RAkkY7N6t3TcE6jE+CIBBbA36lwQ1JyzZs= +cloud.google.com/go v0.62.0/go.mod h1:jmCYTdRCQuc1PHIIJ/maLInMho30T/Y0M4hTdTShOYc= +cloud.google.com/go v0.65.0/go.mod h1:O5N8zS7uWy9vkA9vayVHs65eM1ubvY4h553ofrNHObY= +cloud.google.com/go v0.72.0/go.mod h1:M+5Vjvlc2wnp6tjzE102Dw08nGShTscUx2nZMufOKPI= +cloud.google.com/go v0.74.0/go.mod h1:VV1xSbzvo+9QJOxLDaJfTjx5e+MePCpCWwvftOeQmWk= +cloud.google.com/go v0.78.0/go.mod h1:QjdrLG0uq+YwhjoVOLsS1t7TW8fs36kLs4XO5R5ECHg= +cloud.google.com/go v0.79.0/go.mod h1:3bzgcEeQlzbuEAYu4mrWhKqWjmpprinYgKJLgKHnbb8= +cloud.google.com/go v0.81.0/go.mod h1:mk/AM35KwGk/Nm2YSeZbxXdrNK3KZOYHmLkOqC2V6E0= cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= +cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= +cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc= +cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg= +cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc= +cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ= cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= +cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= cloud.google.com/go/firestore v1.1.0/go.mod h1:ulACoGHTpvq5r8rxGJ4ddJZBZqakUQqClKRT5SZwBmk= cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= +cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= +cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA= +cloud.google.com/go/pubsub v1.3.1/go.mod h1:i+ucay31+CNRpDW4Lu78I4xXG+O1r/MAHgjpRVR+TSU= cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= +cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos= +cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk= +cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs= +cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= github.com/Azure/azure-pipeline-go v0.2.1/go.mod h1:UGSo8XybXnIGZ3epmeBw7Jdz+HiUVpqIlpz/HKHylF4= github.com/Azure/azure-pipeline-go v0.2.2/go.mod h1:4rQ/NZncSvGqNkkOsNpOU1tgoNuIlp9AfUH5G1tvCHc= @@ -39,6 +65,7 @@ github.com/aead/siphash v1.0.1/go.mod h1:Nywa3cDsYNNK3gaciGTWPwHt0wlpNV15vwmswBA github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/allegro/bigcache v1.2.1-0.20190218064605-e24eb225f156/go.mod h1:Cb/ax3seSYIx7SuZdm2G2xzfwmv3TPSk2ucNfQESPXM= +github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= github.com/aristanetworks/goarista v0.0.0-20170210015632-ea17b1a17847/go.mod h1:D/tb0zPVXnP7fmsLZjtdUhSsumbK/ij54UXjjVgMGxQ= github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= @@ -46,9 +73,8 @@ github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmV github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= github.com/aws/aws-sdk-go v1.25.48/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= -github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= -github.com/bketelsen/crypt v0.0.3-0.20200106085610-5cbc8cc4026c/go.mod h1:MKsuJmJgSg28kpZDP6UIiPt0e0Oz0kqKNGyRaWEPv84= +github.com/bketelsen/crypt v0.0.4/go.mod h1:aI6NrJ0pMGgvZKL1iVgXLnfIFJtfV+bKCoqOes/6LfM= github.com/btcsuite/btcd v0.0.0-20171128150713-2e60448ffcc6/go.mod h1:Dmm/EzmjnCiweXmzRIAiUWCInVmPgjkzgv5k4tVyXiQ= github.com/btcsuite/btcd v0.0.0-20190315201642-aa6e0f35703c/go.mod h1:DrZx5ec/dmnfpw9KyYoQyYo7d0KEvTkk/5M/vbZjAr8= github.com/btcsuite/btcd v0.20.1-beta/go.mod h1:wVuoA8VJLEcwgqHBwHmzLRazpKxTv13Px/pDuV7OomQ= @@ -72,18 +98,21 @@ github.com/cespare/cp v0.1.0/go.mod h1:SOGHArjBr4JWaSDEVpWpo/hNg6RoKrls6Oh40hiwW github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= +github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= +github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cloudflare/cloudflare-go v0.10.2-0.20190916151808-a80f83b9add9/go.mod h1:1MxXX1Ux4x6mqPmjkUgTP1CdXIBXKX7T+Jk9Gxrmx+U= +github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= +github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= +github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= github.com/coinbase/rosetta-sdk-go v0.7.0 h1:lmTO/JEpCvZgpbkOITL95rA80CPKb5CtMzLaqF2mCNg= github.com/coinbase/rosetta-sdk-go v0.7.0/go.mod h1:7nD3oBPIiHqhRprqvMgPoGxe/nyq3yftRmpsy29coWE= -github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= -github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk= github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= -github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= -github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= +github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE= github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= @@ -110,7 +139,12 @@ github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4 github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/dvyukov/go-fuzz v0.0.0-20200318091601-be3528f3a813/go.mod h1:11Gm+ccJnvAhCNLlf5+cS9KjtbaD5I5zaZpFMsTHWTw= github.com/edsrzf/mmap-go v0.0.0-20160512033002-935e0e8a636c/go.mod h1:YO35OhQPt3KJa3ryjFM5Bs14WD66h8eGKpfaBNrHW5M= +github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= +github.com/envoyproxy/go-control-plane v0.9.7/go.mod h1:cwu0lG7PUMfa9snN8LXBig5ynNVH9qI8YYLbd1fK2po= +github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= +github.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/ethereum/go-ethereum v1.9.25 h1:mMiw/zOOtCLdGLWfcekua0qPrJTe7FVIiHJ4IKNTfR0= github.com/ethereum/go-ethereum v1.9.25/go.mod h1:vMkFiYLHI4tgPw4k2j4MHKoovchFE8plZ0M9VMk4/oM= @@ -125,30 +159,46 @@ github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4 github.com/gballet/go-libpcsclite v0.0.0-20190607065134-2772fd86a8ff/go.mod h1:x7DCsMOv1taUwEWCzT4cmDeAkigA5/QCwUodaVOe8Ww= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= +github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= +github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= -github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= github.com/go-ole/go-ole v1.2.1/go.mod h1:7FAglXiTm7HKlQRDeOQ6ZNUHidzCWXuZWq/1dTyBNF8= github.com/go-sourcemap/sourcemap v2.1.2+incompatible/go.mod h1:F8jJfvm2KbVjc5NqelyYJmf/v5J0dwNLS2mL4sNA1Jg= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= +github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= -github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= +github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= -github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= +github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= +github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= +github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= +github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4= +github.com/golang/mock v1.5.0/go.mod h1:CWnOUgYIOo4TcNZ0wHX3YZCqsaM1I1Jvs6v3mP3KVu8= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= +github.com/golang/protobuf v1.3.4/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= +github.com/golang/protobuf v1.3.5/go.mod h1:6O5/vntMXwX2lRkT1hjjk0nAC1IDOTvTlVgjlRvqsdk= github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= -github.com/golang/protobuf v1.4.2 h1:+Z5KGCizgyZCbGh1KZqA0fcLLkwbsjIzS4aV2v7wJX0= github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= +github.com/golang/protobuf v1.5.1/go.mod h1:DopwsBzvsk0Fs44TXzsVbJyPhcCPeIwnvohx4u74HPM= +github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw= +github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.3-0.20201103224600-674baa8c7fc3 h1:ur2rms48b3Ep1dxh7aUV2FZEQ8jEVO2F6ILKx8ofkAg= github.com/golang/snappy v0.0.3-0.20201103224600-674baa8c7fc3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= @@ -158,26 +208,41 @@ github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5a github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.2 h1:X2ev0eStA3AbceY54o37/0PQ/UWqKEiiO2dKL5OPaFM= +github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU= +github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/gofuzz v1.1.1-0.20200604201612-c04b05f3adfa/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= +github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= +github.com/google/martian/v3 v3.1.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= +github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20201023163331-3e6fc7fc9c4c/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20201203190320-1bf35d6f28c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20210122040257-d980be63207e/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20210226084205-cbba55b83ad5/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= github.com/gorilla/websocket v1.4.1-0.20190629185528-ae1634f6a989/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= github.com/gorilla/websocket v1.4.1/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= -github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/graph-gophers/graphql-go v0.0.0-20191115155744-f33e81362277/go.mod h1:9CQHMSxwO4MprSdzoIEobiHpoLtHm77vfxsvsIN5Vuc= -github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= -github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= -github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= +github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= github.com/hashicorp/consul/api v1.1.0/go.mod h1:VmuI/Lkw1nC05EYQWNKwWGbkg+FbDBtguAZLlVdkD9Q= github.com/hashicorp/consul/sdk v0.1.1/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8= github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= @@ -203,6 +268,8 @@ github.com/holiman/uint256 v1.1.1/go.mod h1:y4ga/t+u+Xwd7CpDgZESaRcWy0I7XMlTMA25 github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/huin/goupnp v1.0.0/go.mod h1:n9v9KO1tAxYH82qOn+UTIFQDmx5n1Zxd/ClZDMX7Bnc= github.com/huin/goutil v0.0.0-20170803182201-1ca381bf3150/go.mod h1:PpLOETDnJ0o3iZrZfqZzyLl6l7F3c6L1oWn7OICBi6o= +github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= +github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/influxdata/influxdb v1.2.3-0.20180221223340-01288bdb0883/go.mod h1:qZna6X/4elxqT3yI9iZYdZrWWdeFOOprn86kgg4+IzY= @@ -211,18 +278,17 @@ github.com/jedisct1/go-minisign v0.0.0-20190909160543-45766022959e/go.mod h1:G1C github.com/jessevdk/go-flags v0.0.0-20141203071132-1679536dcc89/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= -github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= github.com/jrick/logrotate v1.0.0/go.mod h1:LNinyqDIJnpAur+b8yyulnQw/wDuN1+BYKlTRt3OuAQ= -github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= +github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= +github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/julienschmidt/httprouter v1.1.1-0.20170430222011-975b5c4c7c21/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= -github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/karalabe/usb v0.0.0-20190919080040-51dc0efba356/go.mod h1:Od972xHfMJowv7NGVDiWVxk2zxnWgjLlJzE+F4F7AGU= -github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= +github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/kkdai/bstream v0.0.0-20161212061736-f391b8402d23/go.mod h1:J+Gs4SYgM6CZQHDETBtE9HaSEkGmuNXF86RwHhHUvq4= -github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.2.0 h1:s5hAObm+yFO5uHYt5dYjxi2rXrsnmRpJx4OYvIWUaQs= @@ -234,7 +300,7 @@ github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+ github.com/lucasjones/reggen v0.0.0-20180717132126-cdb49ff09d77 h1:6xiz3+ZczT3M4+I+JLpcPGG1bQKm8067HktB17EDWEE= github.com/lucasjones/reggen v0.0.0-20180717132126-cdb49ff09d77/go.mod h1:5ELEyG+X8f+meRWHuqUOewBOhvHkl7M76pdGEansxW4= github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= -github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= +github.com/magiconair/properties v1.8.5/go.mod h1:y3VJvCyxH9uVvJTWEGAELF3aiYNyPKd5NZ3oSwXrF60= github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= github.com/mattn/go-colorable v0.1.0/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= github.com/mattn/go-colorable v0.1.8/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= @@ -260,11 +326,12 @@ github.com/mitchellh/gox v0.4.0/go.mod h1:Sd9lOJ0+aimLBi73mGofS1ycjY8lL3uZM3JPS4 github.com/mitchellh/iochan v1.0.0/go.mod h1:JwYml1nuB7xOzsp52dPpHFffvOCDupsG0QubkSMEySY= github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= -github.com/mitchellh/mapstructure v1.3.3 h1:SzB1nHZ2Xi+17FP0zVQBHIZqvwRN9408fJO8h+eeNA8= github.com/mitchellh/mapstructure v1.3.3/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= -github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/mitchellh/mapstructure v1.4.1 h1:CpVNEelQCZBooIPDn+AR3NpivK/TIKU8bDxdASFVQag= +github.com/mitchellh/mapstructure v1.4.1/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= -github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/naoina/go-stringutil v0.1.0/go.mod h1:XJ2SJL9jCtBh+P9q5btrd/Ylo8XwT/h1USek5+NqSA0= github.com/naoina/toml v0.1.2-0.20170918210437-9fafd6967416/go.mod h1:NBIhNtsFMo3G2szEBne+bO4gS192HuIYRqfvOWb4i1E= github.com/neilotoole/errgroup v0.1.5 h1:DxEGoIfFm5ooGicidR+okiHjoOaGRKFaSxDPVZuuu2I= @@ -286,27 +353,24 @@ github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFSt github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= github.com/pborman/uuid v0.0.0-20170112150404-1b00554d8222/go.mod h1:VyrYX9gd7irzKovcSS6BIIEwPRkP2Wm2m9ufcdFSJ34= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= +github.com/pelletier/go-toml v1.9.3/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= github.com/peterh/liner v1.1.1-0.20190123174540-a2c9a5303de7/go.mod h1:CRroGNssyjTd/qIG2FyxByd2S8JEAZXBl4qUrZf8GS0= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/sftp v1.10.1/go.mod h1:lYOWFsE0bwd1+KfKJaKeuokY15vzFx25BLbzYYoAxZI= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= -github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= -github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= -github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= -github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/tsdb v0.6.2-0.20190402121629-4f204dcbc150/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= -github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= github.com/rjeczalik/notify v0.9.1/go.mod h1:rKwnCoCGeuQnwBtTSPL9Dad03Vh2n40ePRrjvIXnJho= -github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= +github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rs/cors v0.0.0-20160617231935-a62a804a8a00/go.mod h1:gFx+x8UowdsKA9AchylcLynDq+nNFfI8FkUZdN/jGCU= github.com/rs/xhandler v0.0.0-20160618193221-ed27b6fd6521/go.mod h1:RvLn4FgxWubrpZHtQLnOf6EwhN2hEMusxZOhcW9H3UQ= @@ -318,24 +382,25 @@ github.com/segmentio/fasthash v1.0.3 h1:EI9+KE1EwvMLBWwjpRDc+fEM+prwxDYbslddQGtr github.com/segmentio/fasthash v1.0.3/go.mod h1:waKX8l2N8yckOgmSsXJi7x1ZfdKZ4x7KRMzBtS3oedY= github.com/shirou/gopsutil v2.20.5+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA= github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= -github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= -github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI= github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= +github.com/spf13/afero v1.6.0/go.mod h1:Ai8FlHk4v/PARR026UzYexafAt9roJ7LcLMAmO6Z93I= github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= +github.com/spf13/cast v1.3.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU= -github.com/spf13/cobra v1.1.1 h1:KfztREH0tPxJJ+geloSLaAkaPkr4ki2Er5quFV1TDo4= -github.com/spf13/cobra v1.1.1/go.mod h1:WnodtKOvamDL/PwE2M4iKs8aMDBZ5Q5klgD3qfVJQMI= +github.com/spf13/cobra v1.2.1 h1:+KmjbUw1hriSNMF55oPrkZcb27aECyrj8V2ytv7kWDw= +github.com/spf13/cobra v1.2.1/go.mod h1:ExllRjgxM/piMAM+3tAZvg8fsklGAf3tPfi+i8t68Nk= github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= +github.com/spf13/jwalterweatherman v1.1.0/go.mod h1:aNWZUN0dPAAO/Ljvb5BEdw96iTZ0EXowPYD95IqWIGo= github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s= -github.com/spf13/viper v1.7.0/go.mod h1:8WkrPz2fc9jxqZNCJI/76HCieCp4Q8HaLFoCha5qpdg= +github.com/spf13/viper v1.8.1/go.mod h1:o0Pch8wJ9BVSWGQMbra6iw0oQ5oktSIBaujf1rJH9Ns= github.com/status-im/keycard-go v0.0.0-20190316090335-8537d3370df4/go.mod h1:RZLeN1LMWmRsyYjvAu+I6Dm9QmlDaIIt+Y+4Kd7Tp+Q= github.com/steakknife/bloomfilter v0.0.0-20180922174646-6819c0d2a570/go.mod h1:8OR4w3TdeIHIh1g6EMY5p0gVNOovcWC+1vpc7naMuAw= github.com/steakknife/hamming v0.0.0-20180906055917-c99c65617cd3/go.mod h1:hpGUWaI9xL8pRQCTXQgocU38Qw1g0Us7n5PxxTwTCYU= @@ -359,7 +424,6 @@ github.com/tidwall/pretty v1.0.2 h1:Z7S3cePv9Jwm1KwS0513MRaoUe3S01WPbLNV40pwWZU= github.com/tidwall/pretty v1.0.2/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= github.com/tidwall/sjson v1.1.4 h1:bTSsPLdAYF5QNLSwYsKfBKKTnlGbIuhqL3CpRsjzGhg= github.com/tidwall/sjson v1.1.4/go.mod h1:wXpKXu8CtDjKAZ+3DrKY5ROCorDFahq8l0tey/Lx1fg= -github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/tyler-smith/go-bip39 v1.0.1-0.20181017060643-dbb3b84ba2ef/go.mod h1:sJ5fKU0s6JVwZjjcUEX2zFOnvq0ASQ2K9Zr6cf67kNs= github.com/tyler-smith/go-bip39 v1.0.2/go.mod h1:sJ5fKU0s6JVwZjjcUEX2zFOnvq0ASQ2K9Zr6cf67kNs= github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= @@ -369,23 +433,34 @@ github.com/vmihailenco/msgpack/v5 v5.1.4/go.mod h1:C5gboKD0TJPqWDTVTtrQNfRbiBwHZ github.com/vmihailenco/tagparser v0.1.2 h1:gnjoVuB/kljJ5wICEEOpx98oXMWPLj22G67Vbd1qPqc= github.com/vmihailenco/tagparser v0.1.2/go.mod h1:OeAg3pn3UbLjkWt+rN9oFYB6u/cQgqMEUPoW2WPyhdI= github.com/wsddn/go-ecdh v0.0.0-20161211032359-48726bab9208/go.mod h1:IotVbo4F+mw0EzQ08zFqg7pK3FebNXpaMsRy2RT+Ees= -github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= github.com/ybbus/jsonrpc v2.1.2+incompatible/go.mod h1:XJrh1eMSzdIYFbM08flv0wp5G35eRniyeGut1z+LSiE= -go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= +github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= +go.etcd.io/etcd/api/v3 v3.5.0/go.mod h1:cbVKeC6lCfl7j/8jBhAK6aIYO9XOjdptoxU/nLQcPvs= +go.etcd.io/etcd/client/pkg/v3 v3.5.0/go.mod h1:IJHfcCEKxYu1Os13ZdwCwIUTUVGYTSAM3YSwc9/Ac1g= +go.etcd.io/etcd/client/v2 v2.305.0/go.mod h1:h9puh54ZTgAKtEbut2oe9P4L/oqKCVB6xsXlzd7alYQ= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= -go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= -go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= -go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= +go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= +go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E= +go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= +go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= +go.uber.org/zap v1.17.0/go.mod h1:MXVU+bhUf/A7Xi2HNOnopQOrmycQ5Ih87HtOu4q5SSo= golang.org/x/crypto v0.0.0-20170930174604-9419663f5a44/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= -golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190426145343-a29dc8fdc734/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190909091759-094676da4a83/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200115085410-6d4e4cb37c7d/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= @@ -399,6 +474,11 @@ golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxT golang.org/x/exp v0.0.0-20190731235908-ec7cb31e5a56/go.mod h1:JhuoJpWY28nO4Vef9tZUw9qufEGTyX1+7lmHxV5q5G4= golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek= golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY= +golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= +golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= +golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= +golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= +golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= @@ -408,22 +488,31 @@ golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHl golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs= +golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/lint v0.0.0-20201208152925-83fdc39ff7b5/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/lint v0.0.0-20210508222113-6edffad5e616/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= golang.org/x/mobile v0.0.0-20200801112145-973feb4309de/go.mod h1:skQtrUTUwhdJvXM/2KKJzY8pDgNr9I/FOMqDVRPBUS4= golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= +golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= golang.org/x/mod v0.1.1-0.20191209134235-331c550502dd/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/net v0.0.0-20180719180050-a680a1efc54d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181011144130-49bb7cea24b1/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181023162649-9b4f9f5ad519/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= @@ -432,28 +521,63 @@ golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200222125558-5a598a2470a0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200501053045-e0ff5e5a1de5/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200506145744-7e3656a0809f/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200513185701-a91f0712d120/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20200813134508-3edf25e44fcc/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= -golang.org/x/net v0.0.0-20200904194848-62affa334b73 h1:MXfv8rhZWmFeqX3GNZRsd6vOLoaCHjYEX3qkRo3YBUA= golang.org/x/net v0.0.0-20200904194848-62affa334b73/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20201031054903-ff519b6c9102/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20201209123823-ac852fbbde11/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20210316092652-d523dce5a7f4/go.mod h1:RBQZq4jEuRlivfhVLdyRGr576XBO4/greRjx4P4O3yc= +golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4 h1:4nGaVu0QrbjT/AK2PRLuQfQuh6DJve+pELhqTdAj3x0= +golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20200902213428-5d25da1a8d43/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20201109201403-9fd604954f58/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20201208152858-08078c50e5b5/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210218202405-ba52d332ba99/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210220000619-9bb904979d93/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210313182246-cd4f82c27b84/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210402161424-2e8d93401602/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208 h1:qwRHBd0NqMbJxfbotnDhm2ByMI1Shq4Y6oRJo21SGJA= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20210220032951-036812b2e83c h1:5KslGYwFpkhGh+Q16bwMP3cOontH8FOep7tGV86Y7SQ= +golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181026203630-95b1ffbd15a5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -463,25 +587,57 @@ golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200331124033-c3d80250170d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200501052902-10377860bb8e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200511232937-7e40ca221e25/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200519105757-fe76b779f299/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200814200057-3d37ad5750ed/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200824131525-c12d262b63d8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200905004654-be1d3432aa8f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200909081042-eff7692f9009/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210104204734-6f8348627aad/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210220050731-9a76102bfb43/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210305230114-8fe3ee5dd75b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210315160823-c6e025ad8005/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210403161142-5e06dd20ab57/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c h1:F1jZWGFhYfh0Ci55sIpILtKKK8p3i2/krTr0H1rg74I= golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= @@ -499,8 +655,41 @@ golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtn golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191112195655-aa38f8e97acc/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191130070609-6e064ea0cf2d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191216173652-a0e659d51361/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200117012304-6edc0a871e69/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200117161641-43d50277825c/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200122220014-bf1340f18c4a/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200204074204-1cc6d1ef6c74/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200212150539-ea181f53ac56/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200224181240-023911ca70b2/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200227222343-706bc42d1f0d/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= +golang.org/x/tools v0.0.0-20200312045724-11d5b4c81c7d/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= +golang.org/x/tools v0.0.0-20200331025713-a30bf2db82d4/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8= +golang.org/x/tools v0.0.0-20200501065659-ab2804fb9c9d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200512131952-2bc93b1c0c88/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200515010526-7d3b6ebf133d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200618134242-20370b0cb4b2/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20200904185747-39188db58858/go.mod h1:Cj7w3i3Rnn0Xh82ur9kSqwfTHTeVxaDqrfMjpcNT6bE= +golang.org/x/tools v0.0.0-20201110124207-079ba7bd75cd/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20201201161351-ac6f37ff4c2a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20201208233053-a543418bbed2/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20210105154028-b0ab187a4818/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= +golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -511,10 +700,30 @@ google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= +google.golang.org/api v0.14.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= +google.golang.org/api v0.15.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= +google.golang.org/api v0.17.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.18.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.19.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.20.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.22.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.24.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= +google.golang.org/api v0.28.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= +google.golang.org/api v0.29.0/go.mod h1:Lcubydp8VUV7KeIHD9z2Bys/sm/vGKnG1UHuDBSrHWM= +google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz5138Fc= +google.golang.org/api v0.35.0/go.mod h1:/XrVsuzM0rZmrsbjJutiuftIzeuTQcEeaYcSk/mQ1dg= +google.golang.org/api v0.36.0/go.mod h1:+z5ficQTmoYpPn8LCUNVpK5I7hwkpjbcgqA7I34qYtE= +google.golang.org/api v0.40.0/go.mod h1:fYKFpnQN0DsDSKRVRcQSDQNtqWPfM9i+zNPxepjRCQ8= +google.golang.org/api v0.41.0/go.mod h1:RkxM5lITDfTzmyKFPt+wGrCJbVfniCr2ool8kTBzRTU= +google.golang.org/api v0.43.0/go.mod h1:nQsDGjRXMo4lvh5hP0TKqF244gqhGcr/YSIykhUk/94= +google.golang.org/api v0.44.0/go.mod h1:EBOGZqzyhtvMDoxwS97ctnh0zUmYY6CxqXsc1AvkYD8= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= +google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= @@ -524,12 +733,58 @@ google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98 google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8= google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20191115194625-c23dd37a84c9/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20191216164720-4f79533eabd1/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20191230161307-f3c370f40bfb/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20200115191322-ca5a22157cba/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20200122232147-0452cf42e150/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20200204135345-fa8e72b47b90/go.mod h1:GmwEX6Z4W5gMy59cAlVYjN9JhxgbQH6Gn+gFDQe2lzA= +google.golang.org/genproto v0.0.0-20200212174721-66ed5ce911ce/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200224152610-e50cd9704f63/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200228133532-8c2c7df3a383/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200305110556-506484158171/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200312145019-da6875a35672/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200331122359-1ee6d9798940/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200430143042-b979b6f78d84/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200511104702-f5ebc3bea380/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200513103714-09dca8ec2884/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200515170657-fc4c6c6a6587/go.mod h1:YsZOwe1myG/8QRHRsmBRE1LrgQY60beZKjly0O1fX9U= google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= +google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7FcilCzHH/e9qn6dsT145K34l5v+OpcnNgKAAA= +google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20200904004341-0bd0a958aa1d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201109203340-2640f1f9cdfb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201201144952-b05cb90ed32e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201210142538-e3217bee35cc/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201214200347-8c77b98c765d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210222152913-aa3ee6e6a81c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210303154014-9728d6b83eeb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210310155132-4ce2db91004e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210319143718-93e7006c17a6/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210402141018-6c239bbf2bb1/go.mod h1:9lPAdzaEmUacj36I+k7YKbEc5CXzPIeORRgDAUOu28A= +google.golang.org/genproto v0.0.0-20210602131652-f16073e35f0c/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= +google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= +google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.28.0/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKal+60= +google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk= +google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= +google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= +google.golang.org/grpc v1.31.1/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= +google.golang.org/grpc v1.33.1/go.mod h1:fr5YgcSWrqhRRxogOsw7RzIpsmvOZ6IcH4kBYTpR3n0= +google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= +google.golang.org/grpc v1.34.0/go.mod h1:WotjhfgOW/POjDeRt8vscBtXq+2VjORFy659qA51WJ8= +google.golang.org/grpc v1.35.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= +google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= +google.golang.org/grpc v1.36.1/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= +google.golang.org/grpc v1.38.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= @@ -538,8 +793,11 @@ google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzi google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= -google.golang.org/protobuf v1.25.0 h1:Ejskq+SyPohKW+1uil0JJMtmHCgJPJ/qWTxr8qp+R4c= +google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4= google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= +google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= +google.golang.org/protobuf v1.26.0 h1:bxAC2xTBsZGibn2RTntX0oH50xLsqy1OxA9tTL3p/lk= +google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= @@ -547,24 +805,29 @@ gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogR gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= -gopkg.in/ini.v1 v1.51.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= +gopkg.in/ini.v1 v1.62.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/natefinch/npipe.v2 v2.0.0-20160621034901-c1b8fa8bdcce/go.mod h1:5AcXVHNjg+BDxry382+8OKon8SEWiKktQR07RKPsv1c= gopkg.in/olebedev/go-duktape.v3 v3.0.0-20200619000410-60c24ae608a6/go.mod h1:uAJfkITjFhyEEuUfm7bsmCZRbW5WRq8s9EY8HZ6hCns= -gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= gopkg.in/urfave/cli.v1 v1.20.0/go.mod h1:vuBzUtMdQeixQj8LVd+/98pzhxNGQoyuPBlsXHOQNO0= -gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo= +gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo= +gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= +honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= +honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= +rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= +rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= From 46020594cf52cbccdb42a41c11d300bf19f0c6e5 Mon Sep 17 00:00:00 2001 From: Madhur Shrimal Date: Fri, 5 Nov 2021 10:39:18 -0700 Subject: [PATCH 278/376] update rosetta sdk (#249) --- go.mod | 4 +- go.sum | 225 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 227 insertions(+), 2 deletions(-) diff --git a/go.mod b/go.mod index 69381e90..17f98260 100644 --- a/go.mod +++ b/go.mod @@ -3,9 +3,9 @@ module github.com/coinbase/rosetta-cli go 1.16 require ( - github.com/coinbase/rosetta-sdk-go v0.7.0 + github.com/coinbase/rosetta-sdk-go v0.7.1 github.com/fatih/color v1.13.0 - github.com/olekukonko/tablewriter v0.0.2-0.20190409134802-7e037d187b0c + github.com/olekukonko/tablewriter v0.0.5 github.com/spf13/cobra v1.2.1 github.com/stretchr/testify v1.7.0 golang.org/x/sync v0.0.0-20210220032951-036812b2e83c diff --git a/go.sum b/go.sum index 21d064c1..4d22430e 100644 --- a/go.sum +++ b/go.sum @@ -1,11 +1,13 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= +cloud.google.com/go v0.43.0/go.mod h1:BOSR3VbTLkk6FDC/TcffxP4NF/FFBGA5ku+jvKOP7pg= cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU= cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc= cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0= cloud.google.com/go v0.50.0/go.mod h1:r9sluTvynVuxRIOHXQEHMFffphuXHOMZMycpNR5e6To= +cloud.google.com/go v0.51.0/go.mod h1:hWtGJ6gnXH+KgDv+V0zFGDvpi07n3z8ZNj3T1RW0Gcw= cloud.google.com/go v0.52.0/go.mod h1:pXajvRH/6o3+F9jDHZWQ5PbGhn+o8w9qiu/CffaVdO4= cloud.google.com/go v0.53.0/go.mod h1:fp/UouUEsRkN6ryDKNW/Upv/JBKnv6WDthjR6+vze6M= cloud.google.com/go v0.54.0/go.mod h1:1rq2OEkV3YMf6n/9ZvGWI3GWw0VoqH/1x2nd8Is/bPc= @@ -24,6 +26,7 @@ cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvf cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg= cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc= cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ= +cloud.google.com/go/bigtable v1.2.0/go.mod h1:JcVAOl45lrTmQfLj7T6TxyMzIN/3FGGcFm+2xVAli2o= cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= cloud.google.com/go/firestore v1.1.0/go.mod h1:ulACoGHTpvq5r8rxGJ4ddJZBZqakUQqClKRT5SZwBmk= @@ -36,6 +39,7 @@ cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0Zeo cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk= cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs= cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= +collectd.org v0.3.0/go.mod h1:A/8DzQBkF6abtvrT2j/AU/4tiBgJWYyh0y/oB/4MlWE= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= github.com/Azure/azure-pipeline-go v0.2.1/go.mod h1:UGSo8XybXnIGZ3epmeBw7Jdz+HiUVpqIlpz/HKHylF4= github.com/Azure/azure-pipeline-go v0.2.2/go.mod h1:4rQ/NZncSvGqNkkOsNpOU1tgoNuIlp9AfUH5G1tvCHc= @@ -52,38 +56,60 @@ github.com/Azure/go-autorest/logger v0.1.0/go.mod h1:oExouG+K6PryycPJfVSxi/koC6L github.com/Azure/go-autorest/tracing v0.5.0/go.mod h1:r/s2XiOKccPW3HrqB+W0TQzfbtp2fGCgRFtBroKn4Dk= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= +github.com/DATA-DOG/go-sqlmock v1.3.3/go.mod h1:f/Ixk793poVmq4qj/V1dPUg2JEAKC73Q5eFN3EC/SaM= github.com/DataDog/zstd v1.4.1/go.mod h1:1jcaCB/ufaK+sKp1NBhlGmpz41jOoPQ35bpF36t7BBo= github.com/DataDog/zstd v1.4.5 h1:EndNeuB0l9syBZhut0wns3gV1hL8zX8LIu6ZiVHWLIQ= github.com/DataDog/zstd v1.4.5/go.mod h1:1jcaCB/ufaK+sKp1NBhlGmpz41jOoPQ35bpF36t7BBo= +github.com/DataDog/zstd v1.4.8 h1:Rpmta4xZ/MgZnriKNd24iZMhGpP5dvUcs/uqfBapKZY= +github.com/DataDog/zstd v1.4.8/go.mod h1:g4AWEaM3yOg3HYfnJ3YIawPnVdXJh9QME85blwSAmyw= github.com/OneOfOne/xxhash v1.2.2 h1:KMrpdQIwFcEqXDklaen+P1axHaj9BSKzvpUUfnHldSE= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= github.com/StackExchange/wmi v0.0.0-20180116203802-5d049714c4a6/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg= github.com/VictoriaMetrics/fastcache v1.5.7/go.mod h1:ptDBkNMQI4RtmVo8VS/XwRY6RoTu1dAWCbrk+6WsEM8= +github.com/VictoriaMetrics/fastcache v1.6.0/go.mod h1:0qHz5QP0GMX4pfmMA/zt5RgfNuXJrTP0zS7DqpHGGTw= github.com/Zilliqa/gozilliqa-sdk v1.2.1-0.20201201074141-dd0ecada1be6 h1:1d9pzdbkth4D9AX6ndKSl7of3UTV0RYl3z64U2dXMGo= github.com/Zilliqa/gozilliqa-sdk v1.2.1-0.20201201074141-dd0ecada1be6/go.mod h1:eSYp2T6f0apnuW8TzhV3f6Aff2SE8Dwio++U4ha4yEM= github.com/aead/siphash v1.0.1/go.mod h1:Nywa3cDsYNNK3gaciGTWPwHt0wlpNV15vwmswBAUSII= +github.com/ajstarks/svgo v0.0.0-20180226025133-644b8db467af/go.mod h1:K08gAheRH3/J6wwsYMMT4xOr94bZjxIelGM0+d/wbFw= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/allegro/bigcache v1.2.1-0.20190218064605-e24eb225f156/go.mod h1:Cb/ax3seSYIx7SuZdm2G2xzfwmv3TPSk2ucNfQESPXM= +github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883/go.mod h1:rCTlJbsFo29Kk6CurOXKm700vrz8f0KW0JNfpkRJY/8= github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= +github.com/apache/arrow/go/arrow v0.0.0-20191024131854-af6fa24be0db/go.mod h1:VTxUBvSJ3s3eHAg65PNgrsn5BtqCRPdmyXh6rAfdxN0= github.com/aristanetworks/goarista v0.0.0-20170210015632-ea17b1a17847/go.mod h1:D/tb0zPVXnP7fmsLZjtdUhSsumbK/ij54UXjjVgMGxQ= github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= github.com/aws/aws-sdk-go v1.25.48/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= +github.com/aws/aws-sdk-go-v2 v1.2.0/go.mod h1:zEQs02YRBw1DjK0PoJv3ygDYOFTre1ejlJWl8FwAuQo= +github.com/aws/aws-sdk-go-v2/config v1.1.1/go.mod h1:0XsVy9lBI/BCXm+2Tuvt39YmdHwS5unDQmxZOYe8F5Y= +github.com/aws/aws-sdk-go-v2/credentials v1.1.1/go.mod h1:mM2iIjwl7LULWtS6JCACyInboHirisUUdkBPoTHMOUo= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.0.2/go.mod h1:3hGg3PpiEjHnrkrlasTfxFqUsZ2GCk/fMUn4CbKgSkM= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.0.2/go.mod h1:45MfaXZ0cNbeuT0KQ1XJylq8A6+OpVV2E5kvY/Kq+u8= +github.com/aws/aws-sdk-go-v2/service/route53 v1.1.1/go.mod h1:rLiOUrPLW/Er5kRcQ7NkwbjlijluLsrIbu/iyl35RO4= +github.com/aws/aws-sdk-go-v2/service/sso v1.1.1/go.mod h1:SuZJxklHxLAXgLTc1iFXbEWkXs7QRTQpCLGaKIprQW0= +github.com/aws/aws-sdk-go-v2/service/sts v1.1.1/go.mod h1:Wi0EBZwiz/K44YliU0EKxqTCJGUfYTWXrrBwkq736bM= +github.com/aws/smithy-go v1.1.0/go.mod h1:EzMw8dbp/YJL4A5/sbhGddag+NPT7q084agLbB9LgIw= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= +github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= github.com/bketelsen/crypt v0.0.4/go.mod h1:aI6NrJ0pMGgvZKL1iVgXLnfIFJtfV+bKCoqOes/6LfM= +github.com/bmizerany/pat v0.0.0-20170815010413-6226ea591a40/go.mod h1:8rLXio+WjiTceGBHIoTvn60HIbs7Hm7bcHjyrSqYB9c= +github.com/boltdb/bolt v1.3.1/go.mod h1:clJnj/oiGkjum5o1McbSZDSLxVThjynRyGBgiAx27Ps= github.com/btcsuite/btcd v0.0.0-20171128150713-2e60448ffcc6/go.mod h1:Dmm/EzmjnCiweXmzRIAiUWCInVmPgjkzgv5k4tVyXiQ= github.com/btcsuite/btcd v0.0.0-20190315201642-aa6e0f35703c/go.mod h1:DrZx5ec/dmnfpw9KyYoQyYo7d0KEvTkk/5M/vbZjAr8= github.com/btcsuite/btcd v0.20.1-beta/go.mod h1:wVuoA8VJLEcwgqHBwHmzLRazpKxTv13Px/pDuV7OomQ= github.com/btcsuite/btcd v0.21.0-beta h1:At9hIZdJW0s9E/fAz28nrz6AmcNlSVucCH796ZteX1M= github.com/btcsuite/btcd v0.21.0-beta/go.mod h1:ZSWyehm27aAuS9bvkATT+Xte3hjHZ+MRgMY/8NJ7K94= +github.com/btcsuite/btcd v0.22.0-beta h1:LTDpDKUM5EeOFBPM8IXpinEcmZ6FWfNZbE3lfrfdnWo= +github.com/btcsuite/btcd v0.22.0-beta/go.mod h1:9n5ntfhhHQBIhUvlhDvD3Qg6fRUj4jkN0VB8L8svzOA= github.com/btcsuite/btclog v0.0.0-20170628155309-84c8d2346e9f/go.mod h1:TdznJufoqS23FtqVCzL0ZqgP5MqXbb4fg/WgDys70nA= github.com/btcsuite/btcutil v0.0.0-20190207003914-4c204d697803/go.mod h1:+5NJ2+qvTyV9exUAL/rxXi3DcLg2Ts+ymUAY5y4NvMg= github.com/btcsuite/btcutil v0.0.0-20190425235716-9e5f4b9a998d/go.mod h1:+5NJ2+qvTyV9exUAL/rxXi3DcLg2Ts+ymUAY5y4NvMg= github.com/btcsuite/btcutil v1.0.2/go.mod h1:j9HUFwoQRsZL3V4n+qG+CUnEGHOarIxfC3Le2Yhbcts= +github.com/btcsuite/btcutil v1.0.3-0.20201208143702-a53e38424cce/go.mod h1:0DVlHczLPewLcPGEIeUEzfOJhqGPQ0mJJRDBtD307+o= github.com/btcsuite/go-socks v0.0.0-20170105172521-4720035b7bfd/go.mod h1:HHNXQzUsZCxOoE+CPiyCTO6x34Zs86zZUiwtpXoGdtg= github.com/btcsuite/goleveldb v0.0.0-20160330041536-7834afc9e8cd/go.mod h1:F+uVaaLLH7j4eDXPRvw78tMflu7Ie2bzYOH4Y8rRKBY= github.com/btcsuite/goleveldb v1.0.0/go.mod h1:QiK9vBlgftBg6rWQIj6wFzbPfRjiykIEhBH4obrXJ/I= @@ -91,6 +117,7 @@ github.com/btcsuite/snappy-go v0.0.0-20151229074030-0bdef8d06723/go.mod h1:8woku github.com/btcsuite/snappy-go v1.0.0/go.mod h1:8woku9dyThutzjeg+3xrA5iCpBRH8XEEg3lh6TiUghc= github.com/btcsuite/websocket v0.0.0-20150119174127-31079b680792/go.mod h1:ghJtEyQwv5/p4Mg4C0fgbePVuGr935/5ddU9Z3TmDRY= github.com/btcsuite/winsvc v1.0.0/go.mod h1:jsenWakMcC0zFBFurPLEAyrnc/teJEM1O46fmI40EZs= +github.com/c-bata/go-prompt v0.2.2/go.mod h1:VzqtzE2ksDBcdln8G7mk2RX9QyGjH+OVqOCSiVIqS34= github.com/cenkalti/backoff v2.2.1+incompatible h1:tNowT99t7UNflLxfYYSlKYsBpXdEet03Pg2g16Swow4= github.com/cenkalti/backoff v2.2.1+incompatible/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= @@ -103,11 +130,16 @@ github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5P github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cloudflare/cloudflare-go v0.10.2-0.20190916151808-a80f83b9add9/go.mod h1:1MxXX1Ux4x6mqPmjkUgTP1CdXIBXKX7T+Jk9Gxrmx+U= +github.com/cloudflare/cloudflare-go v0.14.0/go.mod h1:EnwdgGMaFOruiPZRFSgn+TsQ3hQ7C/YWzIGLeu5c304= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= github.com/coinbase/rosetta-sdk-go v0.7.0 h1:lmTO/JEpCvZgpbkOITL95rA80CPKb5CtMzLaqF2mCNg= github.com/coinbase/rosetta-sdk-go v0.7.0/go.mod h1:7nD3oBPIiHqhRprqvMgPoGxe/nyq3yftRmpsy29coWE= +github.com/coinbase/rosetta-sdk-go v0.7.1 h1:J1erjPiDnyDIHwscxXSrx9GnnFhzb8zDhx/On2j7JF8= +github.com/coinbase/rosetta-sdk-go v0.7.1/go.mod h1:MZX7tpDNCZOHm1WpydIfNwpHUDXJR1Pt4xeuABqfvQo= +github.com/consensys/bavard v0.1.8-0.20210406032232-f3452dc9b572/go.mod h1:Bpd0/3mZuaj6Sj+PqrmIquiOKy397AKGThQPaGzNXAQ= +github.com/consensys/gnark-crypto v0.4.1-0.20210426202927-39ac3d4b3f1f/go.mod h1:815PAHg3wvysy0SyIqanF8gZ0Y1wjk/hrDHD/iT88+Q= github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk= github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= @@ -116,29 +148,42 @@ github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSV github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE= github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= +github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= +github.com/cyberdelia/templates v0.0.0-20141128023046-ca7fffd4298c/go.mod h1:GyV+0YP4qX0UQ7r2MoYZ+AvYDp12OF5yg4q8rGnyNh4= +github.com/dave/jennifer v1.2.0/go.mod h1:fIb+770HOpJ2fmN9EPPKOqm1vMGhB+TwXKMZhrIygKg= github.com/davecgh/go-spew v0.0.0-20171005155431-ecdeabc65495/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/deckarep/golang-set v0.0.0-20180603214616-504e848d77ea/go.mod h1:93vsz/8Wt4joVM7c2AVqh+YRMiUSc14yDtF28KmMOgQ= github.com/decred/dcrd/lru v1.0.0/go.mod h1:mxKOwFd7lFjN2GZYsiz/ecgqR6kkYAl+0pz0tEMk218= +github.com/deepmap/oapi-codegen v1.6.0/go.mod h1:ryDa9AgbELGeB+YEXE1dR53yAjHwFvE9iAUlWl9Al3M= +github.com/deepmap/oapi-codegen v1.8.2/go.mod h1:YLgSKSDv/bZQB7N4ws6luhozi3cEdRktEqrX88CvjIw= github.com/dgraph-io/badger/v2 v2.2007.2 h1:EjjK0KqwaFMlPin1ajhP943VPENHJdEz1KLIegjaI3k= github.com/dgraph-io/badger/v2 v2.2007.2/go.mod h1:26P/7fbL4kUZVEVKLAKXkBXKOydDmM2p1e+NhhnBCAE= +github.com/dgraph-io/badger/v2 v2.2007.4 h1:TRWBQg8UrlUhaFdco01nO2uXwzKS7zd+HVdwV/GHc4o= +github.com/dgraph-io/badger/v2 v2.2007.4/go.mod h1:vSw/ax2qojzbN6eXHIx6KPKtCSHJN/Uz0X0VPruTIhk= github.com/dgraph-io/ristretto v0.0.3-0.20200630154024-f66de99634de/go.mod h1:KPxhHT9ZxKefz+PCeOGsrHpl1qZ7i70dGTu2u+Ahh6E= github.com/dgraph-io/ristretto v0.0.3 h1:jh22xisGBjrEVnRZ1DVTpBVQm0Xndu8sMl0CWDzSIBI= github.com/dgraph-io/ristretto v0.0.3/go.mod h1:KPxhHT9ZxKefz+PCeOGsrHpl1qZ7i70dGTu2u+Ahh6E= github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= +github.com/dgryski/go-bitstream v0.0.0-20180413035011-3522498ce2c8/go.mod h1:VMaSuZ+SZcx/wljOQKvp5srsbCiKDEb6K2wC4+PiBmQ= github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 h1:fAjc9m62+UWV/WAFKLNi6ZS0675eEUC9y3AlwSbQu1Y= github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= github.com/dlclark/regexp2 v1.2.0/go.mod h1:2pZnwuY/m+8K6iRw6wQdMtk+rH5tNGR1i55kozfMjCc= +github.com/dlclark/regexp2 v1.4.1-0.20201116162257-a2a8dda75c91/go.mod h1:2pZnwuY/m+8K6iRw6wQdMtk+rH5tNGR1i55kozfMjCc= github.com/docker/docker v1.4.2-0.20180625184442-8e610b2b55bf/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= github.com/dop251/goja v0.0.0-20200721192441-a695b0cdd498/go.mod h1:Mw6PkjjMXWbTj+nnj4s3QPXq1jaT0s5pC0iFD4+BOAA= +github.com/dop251/goja v0.0.0-20211011172007-d99e4b8cbf48/go.mod h1:R9ET47fwRVRPZnOGvHxxhuZcbrMCuiqOz3Rlrh4KSnk= +github.com/dop251/goja_nodejs v0.0.0-20210225215109-d91c329300e7/go.mod h1:hn7BA7c8pLvoGndExHudxTDKZ84Pyvv+90pbBjbTz0Y= github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo= github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/dvyukov/go-fuzz v0.0.0-20200318091601-be3528f3a813/go.mod h1:11Gm+ccJnvAhCNLlf5+cS9KjtbaD5I5zaZpFMsTHWTw= +github.com/eclipse/paho.mqtt.golang v1.2.0/go.mod h1:H9keYFcgq3Qr5OUJm/JZI/i6U7joQ8SYLhZwfeOo6Ts= github.com/edsrzf/mmap-go v0.0.0-20160512033002-935e0e8a636c/go.mod h1:YO35OhQPt3KJa3ryjFM5Bs14WD66h8eGKpfaBNrHW5M= +github.com/edsrzf/mmap-go v1.0.0/go.mod h1:YO35OhQPt3KJa3ryjFM5Bs14WD66h8eGKpfaBNrHW5M= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= @@ -148,27 +193,45 @@ github.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d/go.m github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/ethereum/go-ethereum v1.9.25 h1:mMiw/zOOtCLdGLWfcekua0qPrJTe7FVIiHJ4IKNTfR0= github.com/ethereum/go-ethereum v1.9.25/go.mod h1:vMkFiYLHI4tgPw4k2j4MHKoovchFE8plZ0M9VMk4/oM= +github.com/ethereum/go-ethereum v1.10.11 h1:KKIcwpmur9iTaVbR2dxlHu+peHVhU+/KX//NWvT1n9U= +github.com/ethereum/go-ethereum v1.10.11/go.mod h1:W3yfrFyL9C1pHcwY5hmRHVDaorTiQxhYBkKyu5mEDHw= github.com/fatih/color v1.3.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/fatih/color v1.10.0/go.mod h1:ELkj/draVOlAH/xkhN6mQ50Qd0MPOk5AAr3maGEBuJM= github.com/fatih/color v1.13.0 h1:8LOYc1KYPPmyKMuN8QV2DNRWNbLo6LZ0iLs8+mlH53w= github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk= github.com/fjl/memsize v0.0.0-20180418122429-ca190fb6ffbc/go.mod h1:VvhXpOYNQvB+uIk2RvXzuaQtkQJzzIx6lSBe1xv7hi0= +github.com/fjl/memsize v0.0.0-20190710130421-bcb5799ab5e5/go.mod h1:VvhXpOYNQvB+uIk2RvXzuaQtkQJzzIx6lSBe1xv7hi0= +github.com/fogleman/gg v1.2.1-0.20190220221249-0403632d5b90/go.mod h1:R/bRT+9gY/C5z7JzPU0zXsXHKM4/ayA+zqcVNZzPa1k= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= github.com/gballet/go-libpcsclite v0.0.0-20190607065134-2772fd86a8ff/go.mod h1:x7DCsMOv1taUwEWCzT4cmDeAkigA5/QCwUodaVOe8Ww= +github.com/getkin/kin-openapi v0.53.0/go.mod h1:7Yn5whZr5kJi6t+kShccXS8ae1APpYTW6yheSwk8Yi4= +github.com/getkin/kin-openapi v0.61.0/go.mod h1:7Yn5whZr5kJi6t+kShccXS8ae1APpYTW6yheSwk8Yi4= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= +github.com/glycerine/go-unsnap-stream v0.0.0-20180323001048-9f0cb55181dd/go.mod h1:/20jfyN9Y5QPEAprSgKAUr+glWDY39ZiUEAYOEv5dsE= +github.com/glycerine/goconvey v0.0.0-20190410193231-58a59202ab31/go.mod h1:Ogl1Tioa0aV7gstGFO7KhffUsb9M4ydbEbbxpcEDc24= +github.com/go-chi/chi/v5 v5.0.0/go.mod h1:BBug9lr0cqtdAhsu6R4AAdvufI0/XBzAQSsUqJpoZOs= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= +github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= github.com/go-ole/go-ole v1.2.1/go.mod h1:7FAglXiTm7HKlQRDeOQ6ZNUHidzCWXuZWq/1dTyBNF8= +github.com/go-openapi/jsonpointer v0.19.5/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= +github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= github.com/go-sourcemap/sourcemap v2.1.2+incompatible/go.mod h1:F8jJfvm2KbVjc5NqelyYJmf/v5J0dwNLS2mL4sNA1Jg= +github.com/go-sourcemap/sourcemap v2.1.3+incompatible/go.mod h1:F8jJfvm2KbVjc5NqelyYJmf/v5J0dwNLS2mL4sNA1Jg= +github.com/go-sql-driver/mysql v1.4.1/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= +github.com/gofrs/uuid v3.3.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= +github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= +github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0/go.mod h1:E/TSTwGwJL78qG/PmXZO1EjYhfJinVAhrmmHX6Z8B9k= +github.com/golang/geo v0.0.0-20190916061304-5b978397cfec/go.mod h1:QZ0nwyI2jOfgRAoBvP+ab5aRr7c9x7lhGEJrKvBwjWI= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= @@ -199,11 +262,17 @@ github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaS github.com/golang/protobuf v1.5.1/go.mod h1:DopwsBzvsk0Fs44TXzsVbJyPhcCPeIwnvohx4u74HPM= github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw= github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.3-0.20201103224600-674baa8c7fc3 h1:ur2rms48b3Ep1dxh7aUV2FZEQ8jEVO2F6ILKx8ofkAg= github.com/golang/snappy v0.0.3-0.20201103224600-674baa8c7fc3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= +github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/golangci/lint-1 v0.0.0-20181222135242-d2cdd8c08219/go.mod h1:/X8TswGSh1pIozq4ZwCfxS0WA5JGXguxk94ar/4c87Y= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= +github.com/google/flatbuffers v1.11.0/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= @@ -235,17 +304,21 @@ github.com/google/pprof v0.0.0-20210226084205-cbba55b83ad5/go.mod h1:kpwsk12EmLe github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.1.5/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= github.com/gorilla/websocket v1.4.1-0.20190629185528-ae1634f6a989/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= github.com/gorilla/websocket v1.4.1/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= +github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/graph-gophers/graphql-go v0.0.0-20191115155744-f33e81362277/go.mod h1:9CQHMSxwO4MprSdzoIEobiHpoLtHm77vfxsvsIN5Vuc= +github.com/graph-gophers/graphql-go v0.0.0-20201113091052-beb923fada29/go.mod h1:9CQHMSxwO4MprSdzoIEobiHpoLtHm77vfxsvsIN5Vuc= github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= github.com/hashicorp/consul/api v1.1.0/go.mod h1:VmuI/Lkw1nC05EYQWNKwWGbkg+FbDBtguAZLlVdkD9Q= github.com/hashicorp/consul/sdk v0.1.1/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8= github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= +github.com/hashicorp/go-bexpr v0.1.10/go.mod h1:oxlubA2vC/gFVfX1A6JGp7ls7uCDlfJn732ehYYg+g0= github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM= @@ -259,63 +332,112 @@ github.com/hashicorp/go.net v0.0.1/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= +github.com/hashicorp/golang-lru v0.5.5-0.20210104140557-80c98217689d/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64= github.com/hashicorp/mdns v1.0.0/go.mod h1:tL+uN++7HEJ6SQLQ2/p+z2pH24WQKWjBPkE0mNTz8vQ= github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I= github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc= +github.com/holiman/bloomfilter/v2 v2.0.3/go.mod h1:zpoh+gs7qcpqrHr3dB55AMiJwo0iURXE7ZOP9L9hSkA= github.com/holiman/uint256 v1.1.1/go.mod h1:y4ga/t+u+Xwd7CpDgZESaRcWy0I7XMlTMA25ApIH5Jw= +github.com/holiman/uint256 v1.2.0/go.mod h1:y4ga/t+u+Xwd7CpDgZESaRcWy0I7XMlTMA25ApIH5Jw= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/huin/goupnp v1.0.0/go.mod h1:n9v9KO1tAxYH82qOn+UTIFQDmx5n1Zxd/ClZDMX7Bnc= +github.com/huin/goupnp v1.0.2/go.mod h1:0dxJBVBHqTMjIUMkESDTNgOOx/Mw5wYIfyFmdzSamkM= github.com/huin/goutil v0.0.0-20170803182201-1ca381bf3150/go.mod h1:PpLOETDnJ0o3iZrZfqZzyLl6l7F3c6L1oWn7OICBi6o= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= +github.com/influxdata/flux v0.65.1/go.mod h1:J754/zds0vvpfwuq7Gc2wRdVwEodfpCFM7mYlOw2LqY= github.com/influxdata/influxdb v1.2.3-0.20180221223340-01288bdb0883/go.mod h1:qZna6X/4elxqT3yI9iZYdZrWWdeFOOprn86kgg4+IzY= +github.com/influxdata/influxdb v1.8.3/go.mod h1:JugdFhsvvI8gadxOI6noqNeeBHvWNTbfYGtiAn+2jhI= +github.com/influxdata/influxdb-client-go/v2 v2.4.0/go.mod h1:vLNHdxTJkIf2mSLvGrpj8TCcISApPoXkaxP8g9uRlW8= +github.com/influxdata/influxql v1.1.1-0.20200828144457-65d3ef77d385/go.mod h1:gHp9y86a/pxhjJ+zMjNXiQAA197Xk9wLxaz+fGG+kWk= +github.com/influxdata/line-protocol v0.0.0-20180522152040-32c6aa80de5e/go.mod h1:4kt73NQhadE3daL3WhR5EJ/J2ocX0PZzwxQ0gXJ7oFE= +github.com/influxdata/line-protocol v0.0.0-20200327222509-2487e7298839/go.mod h1:xaLFMmpvUxqXtVkUJfg9QmT88cDaCJ3ZKgdZ78oO8Qo= +github.com/influxdata/line-protocol v0.0.0-20210311194329-9aa0e372d097/go.mod h1:xaLFMmpvUxqXtVkUJfg9QmT88cDaCJ3ZKgdZ78oO8Qo= +github.com/influxdata/promql/v2 v2.12.0/go.mod h1:fxOPu+DY0bqCTCECchSRtWfc+0X19ybifQhZoQNF5D8= +github.com/influxdata/roaring v0.4.13-0.20180809181101-fc520f41fab6/go.mod h1:bSgUQ7q5ZLSO+bKBGqJiCBGAl+9DxyW63zLTujjUlOE= +github.com/influxdata/tdigest v0.0.0-20181121200506-bf2b5ad3c0a9/go.mod h1:Js0mqiSBE6Ffsg94weZZ2c+v/ciT8QRHFOap7EKDrR0= +github.com/influxdata/usage-client v0.0.0-20160829180054-6d3895376368/go.mod h1:Wbbw6tYNvwa5dlB6304Sd+82Z3f7PmVZHVKU637d4po= github.com/jackpal/go-nat-pmp v1.0.2-0.20160603034137-1fa385a6f458/go.mod h1:QPH045xvCAeXUZOxsnwmrtiCoxIr9eob+4orBN1SBKc= github.com/jedisct1/go-minisign v0.0.0-20190909160543-45766022959e/go.mod h1:G1CVv03EnqU1wYL2dFwXxW2An0az9JTl/ZsqXQeBlkU= github.com/jessevdk/go-flags v0.0.0-20141203071132-1679536dcc89/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= +github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= +github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= github.com/jrick/logrotate v1.0.0/go.mod h1:LNinyqDIJnpAur+b8yyulnQw/wDuN1+BYKlTRt3OuAQ= +github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= +github.com/jsternberg/zap-logfmt v1.0.0/go.mod h1:uvPs/4X51zdkcm5jXl5SYoN+4RK21K8mysFmDaM/h+o= github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/julienschmidt/httprouter v1.1.1-0.20170430222011-975b5c4c7c21/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= +github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= +github.com/jung-kurt/gofpdf v1.0.3-0.20190309125859-24315acbbda5/go.mod h1:7Id9E/uU8ce6rXgefFLlgrJj/GYY22cpxn+r32jIOes= +github.com/jwilder/encoding v0.0.0-20170811194829-b4e1701a28ef/go.mod h1:Ct9fl0F6iIOGgxJ5npU/IUOhOhqlVrGjyIZc8/MagT0= github.com/karalabe/usb v0.0.0-20190919080040-51dc0efba356/go.mod h1:Od972xHfMJowv7NGVDiWVxk2zxnWgjLlJzE+F4F7AGU= +github.com/karalabe/usb v0.0.0-20211005121534-4c5740d64559/go.mod h1:Od972xHfMJowv7NGVDiWVxk2zxnWgjLlJzE+F4F7AGU= +github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/kkdai/bstream v0.0.0-20161212061736-f391b8402d23/go.mod h1:J+Gs4SYgM6CZQHDETBtE9HaSEkGmuNXF86RwHhHUvq4= +github.com/klauspost/compress v1.4.0/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= +github.com/klauspost/compress v1.12.3 h1:G5AfA94pHPysR56qqrkO2pxEexdDzrpFJ6yt/VqWxVU= +github.com/klauspost/compress v1.12.3/go.mod h1:8dP1Hq4DHOhN9w426knH3Rhby4rFm6D8eO+e+Dq5Gzg= +github.com/klauspost/cpuid v0.0.0-20170728055534-ae7887de9fa5/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= +github.com/klauspost/crc32 v0.0.0-20161016154125-cb6bfca970f6/go.mod h1:+ZoRqAPRLkC4NPOvfYeR5KNOrY6TD+/sAC3HXPZgDYg= +github.com/klauspost/pgzip v1.0.2-0.20170402124221-0bf5dcad4ada/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs= +github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.2.0 h1:s5hAObm+yFO5uHYt5dYjxi2rXrsnmRpJx4OYvIWUaQs= github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= +github.com/labstack/echo/v4 v4.2.1/go.mod h1:AA49e0DZ8kk5jTOOCKNuPR6oTnBS0dYiM4FW1e6jwpg= +github.com/labstack/gommon v0.3.0/go.mod h1:MULnywXg0yavhxWKc+lOruYdAhDwPK9wf0OL7NoOu+k= +github.com/leanovate/gopter v0.2.9/go.mod h1:U2L/78B+KVFIx2VmW6onHJQzXtFb+p5y3y2Sh+Jxxv8= +github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/lucasjones/reggen v0.0.0-20180717132126-cdb49ff09d77 h1:6xiz3+ZczT3M4+I+JLpcPGG1bQKm8067HktB17EDWEE= github.com/lucasjones/reggen v0.0.0-20180717132126-cdb49ff09d77/go.mod h1:5ELEyG+X8f+meRWHuqUOewBOhvHkl7M76pdGEansxW4= github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/magiconair/properties v1.8.5/go.mod h1:y3VJvCyxH9uVvJTWEGAELF3aiYNyPKd5NZ3oSwXrF60= +github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= +github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= +github.com/matryer/moq v0.0.0-20190312154309-6cfb0558e1bd/go.mod h1:9ELz6aaclSIGnZBoaSLZ3NAl1VTufbOrXBPvtcy6WiQ= github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= github.com/mattn/go-colorable v0.1.0/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= +github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= +github.com/mattn/go-colorable v0.1.7/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= github.com/mattn/go-colorable v0.1.8/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= github.com/mattn/go-colorable v0.1.9 h1:sqDoxXbdeALODt0DAeJCVp38ps9ZogZEAXjus69YV3U= github.com/mattn/go-colorable v0.1.9/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= github.com/mattn/go-ieproxy v0.0.0-20190610004146-91bb50d98149/go.mod h1:31jz6HNzdxOmlERGGEc4v/dMssOfmp2p5bT/okiKFFc= github.com/mattn/go-ieproxy v0.0.0-20190702010315-6dee0af9227d/go.mod h1:31jz6HNzdxOmlERGGEc4v/dMssOfmp2p5bT/okiKFFc= github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= +github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-isatty v0.0.5-0.20180830101745-3fb116b82035/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= +github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= +github.com/mattn/go-isatty v0.0.9/go.mod h1:YNRxwqDuOph6SZLI9vUUz6OYw3QyUt7WiY2yME+cCiQ= github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= github.com/mattn/go-isatty v0.0.14 h1:yVuAays6BHfxijgZPzw+3Zlu5yQgKGP2/hcQbHb7S9Y= github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= github.com/mattn/go-runewidth v0.0.3/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= github.com/mattn/go-runewidth v0.0.4 h1:2BvfKmzob6Bmd4YsL0zygOqfdFnK7GR4QL06Do4/p7Y= github.com/mattn/go-runewidth v0.0.4/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= +github.com/mattn/go-runewidth v0.0.9 h1:Lm995f3rfxdpd6TSmuVCHVb/QhupuXlYr8sCI/QdE+0= +github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= +github.com/mattn/go-sqlite3 v1.11.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= +github.com/mattn/go-tty v0.0.0-20180907095812-13ff1204f104/go.mod h1:XPvLUNfbS4fJH25nqRHfWLMa1ONC8Amw+mIA639KxkE= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc= @@ -329,18 +451,28 @@ github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh github.com/mitchellh/mapstructure v1.3.3/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/mitchellh/mapstructure v1.4.1 h1:CpVNEelQCZBooIPDn+AR3NpivK/TIKU8bDxdASFVQag= github.com/mitchellh/mapstructure v1.4.1/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= +github.com/mitchellh/mapstructure v1.4.2 h1:6h7AQ0yhTcIsmFmnAwQls75jp2Gzs4iB8W7pjMO+rqo= +github.com/mitchellh/mapstructure v1.4.2/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= +github.com/mitchellh/pointerstructure v1.2.0/go.mod h1:BRAsLI5zgXmw97Lf6s25bs8ohIXc3tViBH44KcwB2g4= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/mschoch/smat v0.0.0-20160514031455-90eadee771ae/go.mod h1:qAyveg+e4CE+eKJXWVjKXM4ck2QobLqTDytGJbLLhJg= +github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/naoina/go-stringutil v0.1.0/go.mod h1:XJ2SJL9jCtBh+P9q5btrd/Ylo8XwT/h1USek5+NqSA0= github.com/naoina/toml v0.1.2-0.20170918210437-9fafd6967416/go.mod h1:NBIhNtsFMo3G2szEBne+bO4gS192HuIYRqfvOWb4i1E= github.com/neilotoole/errgroup v0.1.5 h1:DxEGoIfFm5ooGicidR+okiHjoOaGRKFaSxDPVZuuu2I= github.com/neilotoole/errgroup v0.1.5/go.mod h1:Q2nLGf+594h0CLBs/Mbg6qOr7GtqDK7C2S41udRnToE= +github.com/neilotoole/errgroup v0.1.6 h1:PODGqPXdT5BC/zCYIMoTrwV+ujKcW+gBXM6Ye9Ve3R8= +github.com/neilotoole/errgroup v0.1.6/go.mod h1:Q2nLGf+594h0CLBs/Mbg6qOr7GtqDK7C2S41udRnToE= github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= github.com/olekukonko/tablewriter v0.0.1/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo= github.com/olekukonko/tablewriter v0.0.2-0.20190409134802-7e037d187b0c h1:1RHs3tNxjXGHeul8z2t6H2N2TlAqpKe5yryJztRx4Jk= github.com/olekukonko/tablewriter v0.0.2-0.20190409134802-7e037d187b0c/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo= +github.com/olekukonko/tablewriter v0.0.5 h1:P2Ga83D34wi1o9J6Wh1mRuqd4mF/x/lgBS7N7AbDhec= +github.com/olekukonko/tablewriter v0.0.5/go.mod h1:hPp6KlRPjbx+hW8ykQs1w3UBbZlj6HuIJcUGPhkA7kY= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= @@ -349,30 +481,45 @@ github.com/onsi/gomega v1.4.1/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5 github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= +github.com/opentracing/opentracing-go v1.0.2/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= +github.com/opentracing/opentracing-go v1.0.3-0.20180606204148-bd9c31933947/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= +github.com/paulbellamy/ratecounter v0.2.0/go.mod h1:Hfx1hDpSGoqxkVVpBi/IlYD7kChlfo5C6hzIHwPqfFE= github.com/pborman/uuid v0.0.0-20170112150404-1b00554d8222/go.mod h1:VyrYX9gd7irzKovcSS6BIIEwPRkP2Wm2m9ufcdFSJ34= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= github.com/pelletier/go-toml v1.9.3/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= +github.com/peterh/liner v1.0.1-0.20180619022028-8c1271fcf47f/go.mod h1:xIteQHvHuaLYG9IFj6mSxM0fCKrs34IrEQUhOYuGPHc= github.com/peterh/liner v1.1.1-0.20190123174540-a2c9a5303de7/go.mod h1:CRroGNssyjTd/qIG2FyxByd2S8JEAZXBl4qUrZf8GS0= +github.com/philhofer/fwd v1.0.0/go.mod h1:gk3iGcWd9+svBvR0sR+KPcfE+RNWozjowpeBVG3ZVNU= +github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/sftp v1.10.1/go.mod h1:lYOWFsE0bwd1+KfKJaKeuokY15vzFx25BLbzYYoAxZI= +github.com/pkg/term v0.0.0-20180730021639-bffc007b7fd5/go.mod h1:eCbImbZ95eXtAUIbLAuAVnBnwf83mjf6QIVH8SHYwqQ= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= +github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= +github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= +github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= +github.com/prometheus/common v0.6.0/go.mod h1:eBmuwkDJBwy6iBfxCBob6t6dR6ENT/y+J+Zk0j9GMYc= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= +github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/tsdb v0.6.2-0.20190402121629-4f204dcbc150/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= +github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= +github.com/retailnext/hllpp v1.0.1-0.20180308014038-101a6d2f8b52/go.mod h1:RDpi1RftBQPUCDRw6SmxeaREsAaRKnOclghuzp/WRzc= github.com/rjeczalik/notify v0.9.1/go.mod h1:rKwnCoCGeuQnwBtTSPL9Dad03Vh2n40ePRrjvIXnJho= github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rs/cors v0.0.0-20160617231935-a62a804a8a00/go.mod h1:gFx+x8UowdsKA9AchylcLynDq+nNFfI8FkUZdN/jGCU= +github.com/rs/cors v1.7.0/go.mod h1:gFx+x8UowdsKA9AchylcLynDq+nNFfI8FkUZdN/jGCU= github.com/rs/xhandler v0.0.0-20160618193221-ed27b6fd6521/go.mod h1:RvLn4FgxWubrpZHtQLnOf6EwhN2hEMusxZOhcW9H3UQ= github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= @@ -380,8 +527,13 @@ github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= github.com/segmentio/fasthash v1.0.3 h1:EI9+KE1EwvMLBWwjpRDc+fEM+prwxDYbslddQGtrmhM= github.com/segmentio/fasthash v1.0.3/go.mod h1:waKX8l2N8yckOgmSsXJi7x1ZfdKZ4x7KRMzBtS3oedY= +github.com/segmentio/kafka-go v0.1.0/go.mod h1:X6itGqS9L4jDletMsxZ7Dz+JFWxM6JHfPOCvTvk+EJo= +github.com/segmentio/kafka-go v0.2.0/go.mod h1:X6itGqS9L4jDletMsxZ7Dz+JFWxM6JHfPOCvTvk+EJo= +github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= github.com/shirou/gopsutil v2.20.5+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA= +github.com/shirou/gopsutil v3.21.4-0.20210419000835-c7a38de76ee5+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA= github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= +github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= @@ -391,6 +543,7 @@ github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B github.com/spf13/afero v1.6.0/go.mod h1:Ai8FlHk4v/PARR026UzYexafAt9roJ7LcLMAmO6Z93I= github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= github.com/spf13/cast v1.3.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= +github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU= github.com/spf13/cobra v1.2.1 h1:+KmjbUw1hriSNMF55oPrkZcb27aECyrj8V2ytv7kWDw= github.com/spf13/cobra v1.2.1/go.mod h1:ExllRjgxM/piMAM+3tAZvg8fsklGAf3tPfi+i8t68Nk= @@ -407,6 +560,7 @@ github.com/steakknife/hamming v0.0.0-20180906055917-c99c65617cd3/go.mod h1:hpGUW github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1 h1:2vfRuCMp5sSVIDSqO8oNnWJq7mPa6KVP3iPIwFBuy8A= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.2.0/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= @@ -416,23 +570,46 @@ github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5Cc github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= github.com/syndtr/goleveldb v1.0.1-0.20200815110645-5c35d600f0ca/go.mod h1:u2MKkTVTVJWe5D1rCvame8WqhBd88EuIwODJZ1VHCPM= +github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7/go.mod h1:q4W45IWZaF22tdD+VEXcAWRA037jwmWEB5VWYORlTpc= github.com/tidwall/gjson v1.6.7 h1:Mb1M9HZCRWEcXQ8ieJo7auYyyiSux6w9XN3AdTpxJrE= github.com/tidwall/gjson v1.6.7/go.mod h1:zeFuBCIqD4sN/gmqBzZ4j7Jd6UcA2Fc56x7QFsv+8fI= +github.com/tidwall/gjson v1.10.2/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= +github.com/tidwall/gjson v1.11.0 h1:C16pk7tQNiH6VlCrtIXL1w8GaOsi1X3W8KDkE1BuYd4= +github.com/tidwall/gjson v1.11.0/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= github.com/tidwall/match v1.0.3 h1:FQUVvBImDutD8wJLN6c5eMzWtjgONK9MwIBCOrUJKeE= github.com/tidwall/match v1.0.3/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM= +github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA= +github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM= github.com/tidwall/pretty v1.0.2 h1:Z7S3cePv9Jwm1KwS0513MRaoUe3S01WPbLNV40pwWZU= github.com/tidwall/pretty v1.0.2/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= +github.com/tidwall/pretty v1.2.0 h1:RWIZEg2iJ8/g6fDDYzMpobmaoGh5OLl4AXtGUGPcqCs= +github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= github.com/tidwall/sjson v1.1.4 h1:bTSsPLdAYF5QNLSwYsKfBKKTnlGbIuhqL3CpRsjzGhg= github.com/tidwall/sjson v1.1.4/go.mod h1:wXpKXu8CtDjKAZ+3DrKY5ROCorDFahq8l0tey/Lx1fg= +github.com/tidwall/sjson v1.2.3 h1:5+deguEhHSEjmuICXZ21uSSsXotWMA0orU783+Z7Cp8= +github.com/tidwall/sjson v1.2.3/go.mod h1:5WdjKx3AQMvCJ4RG6/2UYT7dLrGvJUV1x4jdTAyGvZs= +github.com/tinylib/msgp v1.0.2/go.mod h1:+d+yLhGm8mzTaHzB+wgMYrodPfmZrzkirds8fDWklFE= +github.com/tklauser/go-sysconf v0.3.5/go.mod h1:MkWzOF4RMCshBAMXuhXJs64Rte09mITnppBXY/rYEFI= +github.com/tklauser/numcpus v0.2.2/go.mod h1:x3qojaO3uyYt0i56EW/VUYs7uBvdl2fkfZFu0T9wgjM= github.com/tyler-smith/go-bip39 v1.0.1-0.20181017060643-dbb3b84ba2ef/go.mod h1:sJ5fKU0s6JVwZjjcUEX2zFOnvq0ASQ2K9Zr6cf67kNs= github.com/tyler-smith/go-bip39 v1.0.2/go.mod h1:sJ5fKU0s6JVwZjjcUEX2zFOnvq0ASQ2K9Zr6cf67kNs= github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= +github.com/urfave/cli/v2 v2.3.0/go.mod h1:LJmUH05zAU44vOAcrfzZQKsZbVcdbOG8rtL3/XcUArI= +github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= +github.com/valyala/fasttemplate v1.0.1/go.mod h1:UQGH1tvbgY+Nz5t2n7tXsz52dQxojPUpymEIMZ47gx8= +github.com/valyala/fasttemplate v1.2.1/go.mod h1:KHLXt3tVN2HBp8eijSv/kGJopbvo7S+qRAEEKiv+SiQ= github.com/vmihailenco/msgpack/v5 v5.1.4 h1:6K44/cU6dMNGkVTGGuu7ef2NdSRFMhAFGGLfE3cqtHM= github.com/vmihailenco/msgpack/v5 v5.1.4/go.mod h1:C5gboKD0TJPqWDTVTtrQNfRbiBwHZGo8UTqP/9/XvLI= +github.com/vmihailenco/msgpack/v5 v5.3.5 h1:5gO0H1iULLWGhs2H5tbAHIZTV8/cYafcFOr9znI5mJU= +github.com/vmihailenco/msgpack/v5 v5.3.5/go.mod h1:7xyJ9e+0+9SaZT0Wt1RGleJXzli6Q/V5KbhBonMG9jc= github.com/vmihailenco/tagparser v0.1.2 h1:gnjoVuB/kljJ5wICEEOpx98oXMWPLj22G67Vbd1qPqc= github.com/vmihailenco/tagparser v0.1.2/go.mod h1:OeAg3pn3UbLjkWt+rN9oFYB6u/cQgqMEUPoW2WPyhdI= +github.com/vmihailenco/tagparser/v2 v2.0.0 h1:y09buUbR+b5aycVFQs/g70pqKVZNBmxwAhO7/IwNM9g= +github.com/vmihailenco/tagparser/v2 v2.0.0/go.mod h1:Wri+At7QHww0WTrCBeu4J6bNtoV6mEfg5OIWRZA9qds= +github.com/willf/bitset v1.1.3/go.mod h1:RjeCKbqT1RxIR/KWY6phxZiaY1IyutSBfGjNPySAYV4= github.com/wsddn/go-ecdh v0.0.0-20161211032359-48726bab9208/go.mod h1:IotVbo4F+mw0EzQ08zFqg7pK3FebNXpaMsRy2RT+Ees= +github.com/xlab/treeprint v0.0.0-20180616005107-d6fb6747feb6/go.mod h1:ce1O1j6UtZfjr22oyGxGLbauSBp2YVXpARAosm7dHBg= github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= github.com/ybbus/jsonrpc v2.1.2+incompatible/go.mod h1:XJrh1eMSzdIYFbM08flv0wp5G35eRniyeGut1z+LSiE= github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= @@ -450,10 +627,14 @@ go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E= +go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= +go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= +go.uber.org/zap v1.9.1/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= go.uber.org/zap v1.17.0/go.mod h1:MXVU+bhUf/A7Xi2HNOnopQOrmycQ5Ih87HtOu4q5SSo= golang.org/x/crypto v0.0.0-20170930174604-9419663f5a44/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= @@ -468,7 +649,12 @@ golang.org/x/crypto v0.0.0-20200510223506-06a226fb4e37/go.mod h1:LzIPMQfyMNhhGPh golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200820211705-5c72a883971a h1:vclmkQCjlDX5OydZ9wv8rBCcS0QyQY66Mpf/7BZbInM= golang.org/x/crypto v0.0.0-20200820211705-5c72a883971a/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20201221181555-eec23a3978ad/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= +golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= +golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20180807140117-3d87b88a115f/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20190125153040-c74c464bbbf2/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= golang.org/x/exp v0.0.0-20190731235908-ec7cb31e5a56/go.mod h1:JhuoJpWY28nO4Vef9tZUw9qufEGTyX1+7lmHxV5q5G4= @@ -479,6 +665,7 @@ golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u0 golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= +golang.org/x/image v0.0.0-20180708004352-c73c2afc3b81/go.mod h1:ux5Hcp/YLpHSI86hEcLt0YII63i6oz57MZXIpbrjZUs= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= @@ -512,6 +699,7 @@ golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73r golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181011144130-49bb7cea24b1/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181023162649-9b4f9f5ad519/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -520,6 +708,7 @@ golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= +golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= @@ -545,10 +734,13 @@ golang.org/x/net v0.0.0-20201031054903-ff519b6c9102/go.mod h1:sp8m0HH+o8qH0wwXwY golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201209123823-ac852fbbde11/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20210220033124-5f55cee0dc0d/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210316092652-d523dce5a7f4/go.mod h1:RBQZq4jEuRlivfhVLdyRGr576XBO4/greRjx4P4O3yc= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4 h1:4nGaVu0QrbjT/AK2PRLuQfQuh6DJve+pELhqTdAj3x0= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= +golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d h1:20cMwl2fHAzkJMEA+8J4JgqBQcQGzbisXo31MIeenXI= +golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -575,11 +767,14 @@ golang.org/x/sync v0.0.0-20210220032951-036812b2e83c h1:5KslGYwFpkhGh+Q16bwMP3cO golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181026203630-95b1ffbd15a5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -588,12 +783,15 @@ golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200107162124-548cf772de50/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -611,6 +809,7 @@ golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200814200057-3d37ad5750ed/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200824131525-c12d262b63d8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200826173525-f9321e4c35a6/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200905004654-be1d3432aa8f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200909081042-eff7692f9009/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -618,15 +817,23 @@ golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210104204734-6f8348627aad/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210220050731-9a76102bfb43/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210305230114-8fe3ee5dd75b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210315160823-c6e025ad8005/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210316164454-77fc1eacc6aa/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210324051608-47abb6519492/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210403161142-5e06dd20ab57/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210420205809-ac73e9fd8988/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c h1:F1jZWGFhYfh0Ci55sIpILtKKK8p3i2/krTr0H1rg74I= golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210816183151-1e6c022a8912 h1:uCLL3g5wH2xjxVREVuAbP9JM5PPKjRbXKRa6IBjkzmU= +golang.org/x/sys v0.0.0-20210816183151-1e6c022a8912/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -635,11 +842,17 @@ golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20201208040808-7e3f01d25324/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/tools v0.0.0-20180525024113-a5b4c53f6e8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190206041539-40960b6deb8e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= @@ -662,6 +875,7 @@ golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtn golang.org/x/tools v0.0.0-20191130070609-6e064ea0cf2d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191216173652-a0e659d51361/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200108203644-89082a384178/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200117012304-6edc0a871e69/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200117161641-43d50277825c/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200122220014-bf1340f18c4a/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= @@ -695,6 +909,12 @@ golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8T golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +gonum.org/v1/gonum v0.0.0-20180816165407-929014505bf4/go.mod h1:Y+Yx5eoAFn32cQvJDxZx5Dpnq+c3wtXuadVZAcxbbBo= +gonum.org/v1/gonum v0.0.0-20181121035319-3f7ecaa7e8ca/go.mod h1:Y+Yx5eoAFn32cQvJDxZx5Dpnq+c3wtXuadVZAcxbbBo= +gonum.org/v1/gonum v0.6.0/go.mod h1:9mxDZsDKxgMAuccQkewq682L+0eCu4dCN2yonUJTCLU= +gonum.org/v1/netlib v0.0.0-20181029234149-ec6d1f5cefe6/go.mod h1:wa6Ws7BG/ESfp6dHfk7C6KdzKA7wR7u/rKwOGE66zvw= +gonum.org/v1/netlib v0.0.0-20190313105609-8cb42192e0e0/go.mod h1:wa6Ws7BG/ESfp6dHfk7C6KdzKA7wR7u/rKwOGE66zvw= +gonum.org/v1/plot v0.0.0-20190515093506-e2840ee46a6b/go.mod h1:Wt8AAjI+ypCyYX3nZBvf6cAIx93T+c/OS2HFAYskSZc= google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= @@ -729,6 +949,7 @@ google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRn google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190716160619-c506a9f90610/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8= @@ -736,6 +957,7 @@ google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvx google.golang.org/genproto v0.0.0-20191115194625-c23dd37a84c9/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= google.golang.org/genproto v0.0.0-20191216164720-4f79533eabd1/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= google.golang.org/genproto v0.0.0-20191230161307-f3c370f40bfb/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20200108215221-bd8f9a0ef82f/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= google.golang.org/genproto v0.0.0-20200115191322-ca5a22157cba/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= google.golang.org/genproto v0.0.0-20200122232147-0452cf42e150/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= google.golang.org/genproto v0.0.0-20200204135345-fa8e72b47b90/go.mod h1:GmwEX6Z4W5gMy59cAlVYjN9JhxgbQH6Gn+gFDQe2lzA= @@ -803,6 +1025,7 @@ gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/ini.v1 v1.62.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= @@ -828,6 +1051,8 @@ honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWh honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= +honnef.co/go/tools v0.1.3/go.mod h1:NgwopIslSNH47DimFoV78dnkksY2EFtX0ajyb3K/las= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= +rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4= rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= From 65879a37496b5c345b6575cff0bfe368a3d19836 Mon Sep 17 00:00:00 2001 From: Madhur Shrimal Date: Fri, 5 Nov 2021 10:51:05 -0700 Subject: [PATCH 279/376] go mod tidy (#250) --- go.sum | 88 +++------------------------------------------------------- 1 file changed, 4 insertions(+), 84 deletions(-) diff --git a/go.sum b/go.sum index 4d22430e..dc98dc64 100644 --- a/go.sum +++ b/go.sum @@ -57,15 +57,11 @@ github.com/Azure/go-autorest/tracing v0.5.0/go.mod h1:r/s2XiOKccPW3HrqB+W0TQzfbt github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/DATA-DOG/go-sqlmock v1.3.3/go.mod h1:f/Ixk793poVmq4qj/V1dPUg2JEAKC73Q5eFN3EC/SaM= -github.com/DataDog/zstd v1.4.1/go.mod h1:1jcaCB/ufaK+sKp1NBhlGmpz41jOoPQ35bpF36t7BBo= -github.com/DataDog/zstd v1.4.5 h1:EndNeuB0l9syBZhut0wns3gV1hL8zX8LIu6ZiVHWLIQ= -github.com/DataDog/zstd v1.4.5/go.mod h1:1jcaCB/ufaK+sKp1NBhlGmpz41jOoPQ35bpF36t7BBo= github.com/DataDog/zstd v1.4.8 h1:Rpmta4xZ/MgZnriKNd24iZMhGpP5dvUcs/uqfBapKZY= github.com/DataDog/zstd v1.4.8/go.mod h1:g4AWEaM3yOg3HYfnJ3YIawPnVdXJh9QME85blwSAmyw= github.com/OneOfOne/xxhash v1.2.2 h1:KMrpdQIwFcEqXDklaen+P1axHaj9BSKzvpUUfnHldSE= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= github.com/StackExchange/wmi v0.0.0-20180116203802-5d049714c4a6/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg= -github.com/VictoriaMetrics/fastcache v1.5.7/go.mod h1:ptDBkNMQI4RtmVo8VS/XwRY6RoTu1dAWCbrk+6WsEM8= github.com/VictoriaMetrics/fastcache v1.6.0/go.mod h1:0qHz5QP0GMX4pfmMA/zt5RgfNuXJrTP0zS7DqpHGGTw= github.com/Zilliqa/gozilliqa-sdk v1.2.1-0.20201201074141-dd0ecada1be6 h1:1d9pzdbkth4D9AX6ndKSl7of3UTV0RYl3z64U2dXMGo= github.com/Zilliqa/gozilliqa-sdk v1.2.1-0.20201201074141-dd0ecada1be6/go.mod h1:eSYp2T6f0apnuW8TzhV3f6Aff2SE8Dwio++U4ha4yEM= @@ -77,12 +73,10 @@ github.com/allegro/bigcache v1.2.1-0.20190218064605-e24eb225f156/go.mod h1:Cb/ax github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883/go.mod h1:rCTlJbsFo29Kk6CurOXKm700vrz8f0KW0JNfpkRJY/8= github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= github.com/apache/arrow/go/arrow v0.0.0-20191024131854-af6fa24be0db/go.mod h1:VTxUBvSJ3s3eHAg65PNgrsn5BtqCRPdmyXh6rAfdxN0= -github.com/aristanetworks/goarista v0.0.0-20170210015632-ea17b1a17847/go.mod h1:D/tb0zPVXnP7fmsLZjtdUhSsumbK/ij54UXjjVgMGxQ= github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= -github.com/aws/aws-sdk-go v1.25.48/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= github.com/aws/aws-sdk-go-v2 v1.2.0/go.mod h1:zEQs02YRBw1DjK0PoJv3ygDYOFTre1ejlJWl8FwAuQo= github.com/aws/aws-sdk-go-v2/config v1.1.1/go.mod h1:0XsVy9lBI/BCXm+2Tuvt39YmdHwS5unDQmxZOYe8F5Y= github.com/aws/aws-sdk-go-v2/credentials v1.1.1/go.mod h1:mM2iIjwl7LULWtS6JCACyInboHirisUUdkBPoTHMOUo= @@ -98,17 +92,13 @@ github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kB github.com/bketelsen/crypt v0.0.4/go.mod h1:aI6NrJ0pMGgvZKL1iVgXLnfIFJtfV+bKCoqOes/6LfM= github.com/bmizerany/pat v0.0.0-20170815010413-6226ea591a40/go.mod h1:8rLXio+WjiTceGBHIoTvn60HIbs7Hm7bcHjyrSqYB9c= github.com/boltdb/bolt v1.3.1/go.mod h1:clJnj/oiGkjum5o1McbSZDSLxVThjynRyGBgiAx27Ps= -github.com/btcsuite/btcd v0.0.0-20171128150713-2e60448ffcc6/go.mod h1:Dmm/EzmjnCiweXmzRIAiUWCInVmPgjkzgv5k4tVyXiQ= github.com/btcsuite/btcd v0.0.0-20190315201642-aa6e0f35703c/go.mod h1:DrZx5ec/dmnfpw9KyYoQyYo7d0KEvTkk/5M/vbZjAr8= github.com/btcsuite/btcd v0.20.1-beta/go.mod h1:wVuoA8VJLEcwgqHBwHmzLRazpKxTv13Px/pDuV7OomQ= -github.com/btcsuite/btcd v0.21.0-beta h1:At9hIZdJW0s9E/fAz28nrz6AmcNlSVucCH796ZteX1M= -github.com/btcsuite/btcd v0.21.0-beta/go.mod h1:ZSWyehm27aAuS9bvkATT+Xte3hjHZ+MRgMY/8NJ7K94= github.com/btcsuite/btcd v0.22.0-beta h1:LTDpDKUM5EeOFBPM8IXpinEcmZ6FWfNZbE3lfrfdnWo= github.com/btcsuite/btcd v0.22.0-beta/go.mod h1:9n5ntfhhHQBIhUvlhDvD3Qg6fRUj4jkN0VB8L8svzOA= github.com/btcsuite/btclog v0.0.0-20170628155309-84c8d2346e9f/go.mod h1:TdznJufoqS23FtqVCzL0ZqgP5MqXbb4fg/WgDys70nA= github.com/btcsuite/btcutil v0.0.0-20190207003914-4c204d697803/go.mod h1:+5NJ2+qvTyV9exUAL/rxXi3DcLg2Ts+ymUAY5y4NvMg= github.com/btcsuite/btcutil v0.0.0-20190425235716-9e5f4b9a998d/go.mod h1:+5NJ2+qvTyV9exUAL/rxXi3DcLg2Ts+ymUAY5y4NvMg= -github.com/btcsuite/btcutil v1.0.2/go.mod h1:j9HUFwoQRsZL3V4n+qG+CUnEGHOarIxfC3Le2Yhbcts= github.com/btcsuite/btcutil v1.0.3-0.20201208143702-a53e38424cce/go.mod h1:0DVlHczLPewLcPGEIeUEzfOJhqGPQ0mJJRDBtD307+o= github.com/btcsuite/go-socks v0.0.0-20170105172521-4720035b7bfd/go.mod h1:HHNXQzUsZCxOoE+CPiyCTO6x34Zs86zZUiwtpXoGdtg= github.com/btcsuite/goleveldb v0.0.0-20160330041536-7834afc9e8cd/go.mod h1:F+uVaaLLH7j4eDXPRvw78tMflu7Ie2bzYOH4Y8rRKBY= @@ -129,13 +119,10 @@ github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWR github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= -github.com/cloudflare/cloudflare-go v0.10.2-0.20190916151808-a80f83b9add9/go.mod h1:1MxXX1Ux4x6mqPmjkUgTP1CdXIBXKX7T+Jk9Gxrmx+U= github.com/cloudflare/cloudflare-go v0.14.0/go.mod h1:EnwdgGMaFOruiPZRFSgn+TsQ3hQ7C/YWzIGLeu5c304= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= -github.com/coinbase/rosetta-sdk-go v0.7.0 h1:lmTO/JEpCvZgpbkOITL95rA80CPKb5CtMzLaqF2mCNg= -github.com/coinbase/rosetta-sdk-go v0.7.0/go.mod h1:7nD3oBPIiHqhRprqvMgPoGxe/nyq3yftRmpsy29coWE= github.com/coinbase/rosetta-sdk-go v0.7.1 h1:J1erjPiDnyDIHwscxXSrx9GnnFhzb8zDhx/On2j7JF8= github.com/coinbase/rosetta-sdk-go v0.7.1/go.mod h1:MZX7tpDNCZOHm1WpydIfNwpHUDXJR1Pt4xeuABqfvQo= github.com/consensys/bavard v0.1.8-0.20210406032232-f3452dc9b572/go.mod h1:Bpd0/3mZuaj6Sj+PqrmIquiOKy397AKGThQPaGzNXAQ= @@ -159,8 +146,6 @@ github.com/deckarep/golang-set v0.0.0-20180603214616-504e848d77ea/go.mod h1:93vs github.com/decred/dcrd/lru v1.0.0/go.mod h1:mxKOwFd7lFjN2GZYsiz/ecgqR6kkYAl+0pz0tEMk218= github.com/deepmap/oapi-codegen v1.6.0/go.mod h1:ryDa9AgbELGeB+YEXE1dR53yAjHwFvE9iAUlWl9Al3M= github.com/deepmap/oapi-codegen v1.8.2/go.mod h1:YLgSKSDv/bZQB7N4ws6luhozi3cEdRktEqrX88CvjIw= -github.com/dgraph-io/badger/v2 v2.2007.2 h1:EjjK0KqwaFMlPin1ajhP943VPENHJdEz1KLIegjaI3k= -github.com/dgraph-io/badger/v2 v2.2007.2/go.mod h1:26P/7fbL4kUZVEVKLAKXkBXKOydDmM2p1e+NhhnBCAE= github.com/dgraph-io/badger/v2 v2.2007.4 h1:TRWBQg8UrlUhaFdco01nO2uXwzKS7zd+HVdwV/GHc4o= github.com/dgraph-io/badger/v2 v2.2007.4/go.mod h1:vSw/ax2qojzbN6eXHIx6KPKtCSHJN/Uz0X0VPruTIhk= github.com/dgraph-io/ristretto v0.0.3-0.20200630154024-f66de99634de/go.mod h1:KPxhHT9ZxKefz+PCeOGsrHpl1qZ7i70dGTu2u+Ahh6E= @@ -172,17 +157,13 @@ github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUn github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 h1:fAjc9m62+UWV/WAFKLNi6ZS0675eEUC9y3AlwSbQu1Y= github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= -github.com/dlclark/regexp2 v1.2.0/go.mod h1:2pZnwuY/m+8K6iRw6wQdMtk+rH5tNGR1i55kozfMjCc= github.com/dlclark/regexp2 v1.4.1-0.20201116162257-a2a8dda75c91/go.mod h1:2pZnwuY/m+8K6iRw6wQdMtk+rH5tNGR1i55kozfMjCc= github.com/docker/docker v1.4.2-0.20180625184442-8e610b2b55bf/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= -github.com/dop251/goja v0.0.0-20200721192441-a695b0cdd498/go.mod h1:Mw6PkjjMXWbTj+nnj4s3QPXq1jaT0s5pC0iFD4+BOAA= github.com/dop251/goja v0.0.0-20211011172007-d99e4b8cbf48/go.mod h1:R9ET47fwRVRPZnOGvHxxhuZcbrMCuiqOz3Rlrh4KSnk= github.com/dop251/goja_nodejs v0.0.0-20210225215109-d91c329300e7/go.mod h1:hn7BA7c8pLvoGndExHudxTDKZ84Pyvv+90pbBjbTz0Y= github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo= github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= -github.com/dvyukov/go-fuzz v0.0.0-20200318091601-be3528f3a813/go.mod h1:11Gm+ccJnvAhCNLlf5+cS9KjtbaD5I5zaZpFMsTHWTw= github.com/eclipse/paho.mqtt.golang v1.2.0/go.mod h1:H9keYFcgq3Qr5OUJm/JZI/i6U7joQ8SYLhZwfeOo6Ts= -github.com/edsrzf/mmap-go v0.0.0-20160512033002-935e0e8a636c/go.mod h1:YO35OhQPt3KJa3ryjFM5Bs14WD66h8eGKpfaBNrHW5M= github.com/edsrzf/mmap-go v1.0.0/go.mod h1:YO35OhQPt3KJa3ryjFM5Bs14WD66h8eGKpfaBNrHW5M= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= @@ -191,16 +172,12 @@ github.com/envoyproxy/go-control-plane v0.9.7/go.mod h1:cwu0lG7PUMfa9snN8LXBig5y github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= github.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= -github.com/ethereum/go-ethereum v1.9.25 h1:mMiw/zOOtCLdGLWfcekua0qPrJTe7FVIiHJ4IKNTfR0= -github.com/ethereum/go-ethereum v1.9.25/go.mod h1:vMkFiYLHI4tgPw4k2j4MHKoovchFE8plZ0M9VMk4/oM= github.com/ethereum/go-ethereum v1.10.11 h1:KKIcwpmur9iTaVbR2dxlHu+peHVhU+/KX//NWvT1n9U= github.com/ethereum/go-ethereum v1.10.11/go.mod h1:W3yfrFyL9C1pHcwY5hmRHVDaorTiQxhYBkKyu5mEDHw= -github.com/fatih/color v1.3.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/fatih/color v1.10.0/go.mod h1:ELkj/draVOlAH/xkhN6mQ50Qd0MPOk5AAr3maGEBuJM= github.com/fatih/color v1.13.0 h1:8LOYc1KYPPmyKMuN8QV2DNRWNbLo6LZ0iLs8+mlH53w= github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk= -github.com/fjl/memsize v0.0.0-20180418122429-ca190fb6ffbc/go.mod h1:VvhXpOYNQvB+uIk2RvXzuaQtkQJzzIx6lSBe1xv7hi0= github.com/fjl/memsize v0.0.0-20190710130421-bcb5799ab5e5/go.mod h1:VvhXpOYNQvB+uIk2RvXzuaQtkQJzzIx6lSBe1xv7hi0= github.com/fogleman/gg v1.2.1-0.20190220221249-0403632d5b90/go.mod h1:R/bRT+9gY/C5z7JzPU0zXsXHKM4/ayA+zqcVNZzPa1k= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= @@ -221,7 +198,6 @@ github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V github.com/go-ole/go-ole v1.2.1/go.mod h1:7FAglXiTm7HKlQRDeOQ6ZNUHidzCWXuZWq/1dTyBNF8= github.com/go-openapi/jsonpointer v0.19.5/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= -github.com/go-sourcemap/sourcemap v2.1.2+incompatible/go.mod h1:F8jJfvm2KbVjc5NqelyYJmf/v5J0dwNLS2mL4sNA1Jg= github.com/go-sourcemap/sourcemap v2.1.3+incompatible/go.mod h1:F8jJfvm2KbVjc5NqelyYJmf/v5J0dwNLS2mL4sNA1Jg= github.com/go-sql-driver/mysql v1.4.1/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= @@ -263,9 +239,6 @@ github.com/golang/protobuf v1.5.1/go.mod h1:DopwsBzvsk0Fs44TXzsVbJyPhcCPeIwnvohx github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw= github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= -github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= -github.com/golang/snappy v0.0.3-0.20201103224600-674baa8c7fc3 h1:ur2rms48b3Ep1dxh7aUV2FZEQ8jEVO2F6ILKx8ofkAg= -github.com/golang/snappy v0.0.3-0.20201103224600-674baa8c7fc3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= @@ -309,10 +282,8 @@ github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+ github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= -github.com/gorilla/websocket v1.4.1-0.20190629185528-ae1634f6a989/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= github.com/gorilla/websocket v1.4.1/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= -github.com/graph-gophers/graphql-go v0.0.0-20191115155744-f33e81362277/go.mod h1:9CQHMSxwO4MprSdzoIEobiHpoLtHm77vfxsvsIN5Vuc= github.com/graph-gophers/graphql-go v0.0.0-20201113091052-beb923fada29/go.mod h1:9CQHMSxwO4MprSdzoIEobiHpoLtHm77vfxsvsIN5Vuc= github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= github.com/hashicorp/consul/api v1.1.0/go.mod h1:VmuI/Lkw1nC05EYQWNKwWGbkg+FbDBtguAZLlVdkD9Q= @@ -331,7 +302,6 @@ github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/b github.com/hashicorp/go.net v0.0.1/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA0oac0k90= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= -github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= github.com/hashicorp/golang-lru v0.5.5-0.20210104140557-80c98217689d/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64= @@ -339,10 +309,8 @@ github.com/hashicorp/mdns v1.0.0/go.mod h1:tL+uN++7HEJ6SQLQ2/p+z2pH24WQKWjBPkE0m github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I= github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc= github.com/holiman/bloomfilter/v2 v2.0.3/go.mod h1:zpoh+gs7qcpqrHr3dB55AMiJwo0iURXE7ZOP9L9hSkA= -github.com/holiman/uint256 v1.1.1/go.mod h1:y4ga/t+u+Xwd7CpDgZESaRcWy0I7XMlTMA25ApIH5Jw= github.com/holiman/uint256 v1.2.0/go.mod h1:y4ga/t+u+Xwd7CpDgZESaRcWy0I7XMlTMA25ApIH5Jw= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= -github.com/huin/goupnp v1.0.0/go.mod h1:n9v9KO1tAxYH82qOn+UTIFQDmx5n1Zxd/ClZDMX7Bnc= github.com/huin/goupnp v1.0.2/go.mod h1:0dxJBVBHqTMjIUMkESDTNgOOx/Mw5wYIfyFmdzSamkM= github.com/huin/goutil v0.0.0-20170803182201-1ca381bf3150/go.mod h1:PpLOETDnJ0o3iZrZfqZzyLl6l7F3c6L1oWn7OICBi6o= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= @@ -350,7 +318,6 @@ github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1: github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/influxdata/flux v0.65.1/go.mod h1:J754/zds0vvpfwuq7Gc2wRdVwEodfpCFM7mYlOw2LqY= -github.com/influxdata/influxdb v1.2.3-0.20180221223340-01288bdb0883/go.mod h1:qZna6X/4elxqT3yI9iZYdZrWWdeFOOprn86kgg4+IzY= github.com/influxdata/influxdb v1.8.3/go.mod h1:JugdFhsvvI8gadxOI6noqNeeBHvWNTbfYGtiAn+2jhI= github.com/influxdata/influxdb-client-go/v2 v2.4.0/go.mod h1:vLNHdxTJkIf2mSLvGrpj8TCcISApPoXkaxP8g9uRlW8= github.com/influxdata/influxql v1.1.1-0.20200828144457-65d3ef77d385/go.mod h1:gHp9y86a/pxhjJ+zMjNXiQAA197Xk9wLxaz+fGG+kWk= @@ -365,7 +332,6 @@ github.com/jackpal/go-nat-pmp v1.0.2-0.20160603034137-1fa385a6f458/go.mod h1:QPH github.com/jedisct1/go-minisign v0.0.0-20190909160543-45766022959e/go.mod h1:G1CVv03EnqU1wYL2dFwXxW2An0az9JTl/ZsqXQeBlkU= github.com/jessevdk/go-flags v0.0.0-20141203071132-1679536dcc89/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= -github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= github.com/jrick/logrotate v1.0.0/go.mod h1:LNinyqDIJnpAur+b8yyulnQw/wDuN1+BYKlTRt3OuAQ= @@ -375,11 +341,9 @@ github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1 github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= github.com/jsternberg/zap-logfmt v1.0.0/go.mod h1:uvPs/4X51zdkcm5jXl5SYoN+4RK21K8mysFmDaM/h+o= github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= -github.com/julienschmidt/httprouter v1.1.1-0.20170430222011-975b5c4c7c21/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/jung-kurt/gofpdf v1.0.3-0.20190309125859-24315acbbda5/go.mod h1:7Id9E/uU8ce6rXgefFLlgrJj/GYY22cpxn+r32jIOes= github.com/jwilder/encoding v0.0.0-20170811194829-b4e1701a28ef/go.mod h1:Ct9fl0F6iIOGgxJ5npU/IUOhOhqlVrGjyIZc8/MagT0= -github.com/karalabe/usb v0.0.0-20190919080040-51dc0efba356/go.mod h1:Od972xHfMJowv7NGVDiWVxk2zxnWgjLlJzE+F4F7AGU= github.com/karalabe/usb v0.0.0-20211005121534-4c5740d64559/go.mod h1:Od972xHfMJowv7NGVDiWVxk2zxnWgjLlJzE+F4F7AGU= github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= @@ -395,12 +359,11 @@ github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxv github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= -github.com/kr/pretty v0.2.0 h1:s5hAObm+yFO5uHYt5dYjxi2rXrsnmRpJx4OYvIWUaQs= -github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pretty v0.2.1 h1:Fmg33tUaq4/8ym9TJN1x7sLJnHVwhP33CNkpYV/7rwI= github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= -github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= github.com/labstack/echo/v4 v4.2.1/go.mod h1:AA49e0DZ8kk5jTOOCKNuPR6oTnBS0dYiM4FW1e6jwpg= @@ -415,7 +378,6 @@ github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/matryer/moq v0.0.0-20190312154309-6cfb0558e1bd/go.mod h1:9ELz6aaclSIGnZBoaSLZ3NAl1VTufbOrXBPvtcy6WiQ= github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= -github.com/mattn/go-colorable v0.1.0/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= github.com/mattn/go-colorable v0.1.7/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= github.com/mattn/go-colorable v0.1.8/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= @@ -425,15 +387,12 @@ github.com/mattn/go-ieproxy v0.0.0-20190610004146-91bb50d98149/go.mod h1:31jz6HN github.com/mattn/go-ieproxy v0.0.0-20190702010315-6dee0af9227d/go.mod h1:31jz6HNzdxOmlERGGEc4v/dMssOfmp2p5bT/okiKFFc= github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= -github.com/mattn/go-isatty v0.0.5-0.20180830101745-3fb116b82035/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.9/go.mod h1:YNRxwqDuOph6SZLI9vUUz6OYw3QyUt7WiY2yME+cCiQ= github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= github.com/mattn/go-isatty v0.0.14 h1:yVuAays6BHfxijgZPzw+3Zlu5yQgKGP2/hcQbHb7S9Y= github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= github.com/mattn/go-runewidth v0.0.3/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= -github.com/mattn/go-runewidth v0.0.4 h1:2BvfKmzob6Bmd4YsL0zygOqfdFnK7GR4QL06Do4/p7Y= -github.com/mattn/go-runewidth v0.0.4/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= github.com/mattn/go-runewidth v0.0.9 h1:Lm995f3rfxdpd6TSmuVCHVb/QhupuXlYr8sCI/QdE+0= github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= github.com/mattn/go-sqlite3 v1.11.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= @@ -448,8 +407,6 @@ github.com/mitchellh/gox v0.4.0/go.mod h1:Sd9lOJ0+aimLBi73mGofS1ycjY8lL3uZM3JPS4 github.com/mitchellh/iochan v1.0.0/go.mod h1:JwYml1nuB7xOzsp52dPpHFffvOCDupsG0QubkSMEySY= github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= -github.com/mitchellh/mapstructure v1.3.3/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= -github.com/mitchellh/mapstructure v1.4.1 h1:CpVNEelQCZBooIPDn+AR3NpivK/TIKU8bDxdASFVQag= github.com/mitchellh/mapstructure v1.4.1/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/mitchellh/mapstructure v1.4.2 h1:6h7AQ0yhTcIsmFmnAwQls75jp2Gzs4iB8W7pjMO+rqo= github.com/mitchellh/mapstructure v1.4.2/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= @@ -462,15 +419,10 @@ github.com/mschoch/smat v0.0.0-20160514031455-90eadee771ae/go.mod h1:qAyveg+e4CE github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/naoina/go-stringutil v0.1.0/go.mod h1:XJ2SJL9jCtBh+P9q5btrd/Ylo8XwT/h1USek5+NqSA0= github.com/naoina/toml v0.1.2-0.20170918210437-9fafd6967416/go.mod h1:NBIhNtsFMo3G2szEBne+bO4gS192HuIYRqfvOWb4i1E= -github.com/neilotoole/errgroup v0.1.5 h1:DxEGoIfFm5ooGicidR+okiHjoOaGRKFaSxDPVZuuu2I= -github.com/neilotoole/errgroup v0.1.5/go.mod h1:Q2nLGf+594h0CLBs/Mbg6qOr7GtqDK7C2S41udRnToE= github.com/neilotoole/errgroup v0.1.6 h1:PODGqPXdT5BC/zCYIMoTrwV+ujKcW+gBXM6Ye9Ve3R8= github.com/neilotoole/errgroup v0.1.6/go.mod h1:Q2nLGf+594h0CLBs/Mbg6qOr7GtqDK7C2S41udRnToE= github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= -github.com/olekukonko/tablewriter v0.0.1/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo= -github.com/olekukonko/tablewriter v0.0.2-0.20190409134802-7e037d187b0c h1:1RHs3tNxjXGHeul8z2t6H2N2TlAqpKe5yryJztRx4Jk= -github.com/olekukonko/tablewriter v0.0.2-0.20190409134802-7e037d187b0c/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo= github.com/olekukonko/tablewriter v0.0.5 h1:P2Ga83D34wi1o9J6Wh1mRuqd4mF/x/lgBS7N7AbDhec= github.com/olekukonko/tablewriter v0.0.5/go.mod h1:hPp6KlRPjbx+hW8ykQs1w3UBbZlj6HuIJcUGPhkA7kY= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= @@ -486,7 +438,6 @@ github.com/opentracing/opentracing-go v1.0.3-0.20180606204148-bd9c31933947/go.mo github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= github.com/paulbellamy/ratecounter v0.2.0/go.mod h1:Hfx1hDpSGoqxkVVpBi/IlYD7kChlfo5C6hzIHwPqfFE= -github.com/pborman/uuid v0.0.0-20170112150404-1b00554d8222/go.mod h1:VyrYX9gd7irzKovcSS6BIIEwPRkP2Wm2m9ufcdFSJ34= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= github.com/pelletier/go-toml v1.9.3/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= github.com/peterh/liner v1.0.1-0.20180619022028-8c1271fcf47f/go.mod h1:xIteQHvHuaLYG9IFj6mSxM0fCKrs34IrEQUhOYuGPHc= @@ -512,15 +463,12 @@ github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y8 github.com/prometheus/common v0.6.0/go.mod h1:eBmuwkDJBwy6iBfxCBob6t6dR6ENT/y+J+Zk0j9GMYc= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= -github.com/prometheus/tsdb v0.6.2-0.20190402121629-4f204dcbc150/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= github.com/retailnext/hllpp v1.0.1-0.20180308014038-101a6d2f8b52/go.mod h1:RDpi1RftBQPUCDRw6SmxeaREsAaRKnOclghuzp/WRzc= github.com/rjeczalik/notify v0.9.1/go.mod h1:rKwnCoCGeuQnwBtTSPL9Dad03Vh2n40ePRrjvIXnJho= github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= -github.com/rs/cors v0.0.0-20160617231935-a62a804a8a00/go.mod h1:gFx+x8UowdsKA9AchylcLynDq+nNFfI8FkUZdN/jGCU= github.com/rs/cors v1.7.0/go.mod h1:gFx+x8UowdsKA9AchylcLynDq+nNFfI8FkUZdN/jGCU= -github.com/rs/xhandler v0.0.0-20160618193221-ed27b6fd6521/go.mod h1:RvLn4FgxWubrpZHtQLnOf6EwhN2hEMusxZOhcW9H3UQ= github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= @@ -530,7 +478,6 @@ github.com/segmentio/fasthash v1.0.3/go.mod h1:waKX8l2N8yckOgmSsXJi7x1ZfdKZ4x7KR github.com/segmentio/kafka-go v0.1.0/go.mod h1:X6itGqS9L4jDletMsxZ7Dz+JFWxM6JHfPOCvTvk+EJo= github.com/segmentio/kafka-go v0.2.0/go.mod h1:X6itGqS9L4jDletMsxZ7Dz+JFWxM6JHfPOCvTvk+EJo= github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= -github.com/shirou/gopsutil v2.20.5+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA= github.com/shirou/gopsutil v3.21.4-0.20210419000835-c7a38de76ee5+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA= github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= @@ -555,8 +502,6 @@ github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s= github.com/spf13/viper v1.8.1/go.mod h1:o0Pch8wJ9BVSWGQMbra6iw0oQ5oktSIBaujf1rJH9Ns= github.com/status-im/keycard-go v0.0.0-20190316090335-8537d3370df4/go.mod h1:RZLeN1LMWmRsyYjvAu+I6Dm9QmlDaIIt+Y+4Kd7Tp+Q= -github.com/steakknife/bloomfilter v0.0.0-20180922174646-6819c0d2a570/go.mod h1:8OR4w3TdeIHIh1g6EMY5p0gVNOovcWC+1vpc7naMuAw= -github.com/steakknife/hamming v0.0.0-20180906055917-c99c65617cd3/go.mod h1:hpGUWaI9xL8pRQCTXQgocU38Qw1g0Us7n5PxxTwTCYU= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1 h1:2vfRuCMp5sSVIDSqO8oNnWJq7mPa6KVP3iPIwFBuy8A= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= @@ -569,23 +514,14 @@ github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= -github.com/syndtr/goleveldb v1.0.1-0.20200815110645-5c35d600f0ca/go.mod h1:u2MKkTVTVJWe5D1rCvame8WqhBd88EuIwODJZ1VHCPM= github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7/go.mod h1:q4W45IWZaF22tdD+VEXcAWRA037jwmWEB5VWYORlTpc= -github.com/tidwall/gjson v1.6.7 h1:Mb1M9HZCRWEcXQ8ieJo7auYyyiSux6w9XN3AdTpxJrE= -github.com/tidwall/gjson v1.6.7/go.mod h1:zeFuBCIqD4sN/gmqBzZ4j7Jd6UcA2Fc56x7QFsv+8fI= github.com/tidwall/gjson v1.10.2/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= github.com/tidwall/gjson v1.11.0 h1:C16pk7tQNiH6VlCrtIXL1w8GaOsi1X3W8KDkE1BuYd4= github.com/tidwall/gjson v1.11.0/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= -github.com/tidwall/match v1.0.3 h1:FQUVvBImDutD8wJLN6c5eMzWtjgONK9MwIBCOrUJKeE= -github.com/tidwall/match v1.0.3/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM= github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA= github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM= -github.com/tidwall/pretty v1.0.2 h1:Z7S3cePv9Jwm1KwS0513MRaoUe3S01WPbLNV40pwWZU= -github.com/tidwall/pretty v1.0.2/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= github.com/tidwall/pretty v1.2.0 h1:RWIZEg2iJ8/g6fDDYzMpobmaoGh5OLl4AXtGUGPcqCs= github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= -github.com/tidwall/sjson v1.1.4 h1:bTSsPLdAYF5QNLSwYsKfBKKTnlGbIuhqL3CpRsjzGhg= -github.com/tidwall/sjson v1.1.4/go.mod h1:wXpKXu8CtDjKAZ+3DrKY5ROCorDFahq8l0tey/Lx1fg= github.com/tidwall/sjson v1.2.3 h1:5+deguEhHSEjmuICXZ21uSSsXotWMA0orU783+Z7Cp8= github.com/tidwall/sjson v1.2.3/go.mod h1:5WdjKx3AQMvCJ4RG6/2UYT7dLrGvJUV1x4jdTAyGvZs= github.com/tinylib/msgp v1.0.2/go.mod h1:+d+yLhGm8mzTaHzB+wgMYrodPfmZrzkirds8fDWklFE= @@ -594,21 +530,15 @@ github.com/tklauser/numcpus v0.2.2/go.mod h1:x3qojaO3uyYt0i56EW/VUYs7uBvdl2fkfZF github.com/tyler-smith/go-bip39 v1.0.1-0.20181017060643-dbb3b84ba2ef/go.mod h1:sJ5fKU0s6JVwZjjcUEX2zFOnvq0ASQ2K9Zr6cf67kNs= github.com/tyler-smith/go-bip39 v1.0.2/go.mod h1:sJ5fKU0s6JVwZjjcUEX2zFOnvq0ASQ2K9Zr6cf67kNs= github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= -github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= github.com/urfave/cli/v2 v2.3.0/go.mod h1:LJmUH05zAU44vOAcrfzZQKsZbVcdbOG8rtL3/XcUArI= github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= github.com/valyala/fasttemplate v1.0.1/go.mod h1:UQGH1tvbgY+Nz5t2n7tXsz52dQxojPUpymEIMZ47gx8= github.com/valyala/fasttemplate v1.2.1/go.mod h1:KHLXt3tVN2HBp8eijSv/kGJopbvo7S+qRAEEKiv+SiQ= -github.com/vmihailenco/msgpack/v5 v5.1.4 h1:6K44/cU6dMNGkVTGGuu7ef2NdSRFMhAFGGLfE3cqtHM= -github.com/vmihailenco/msgpack/v5 v5.1.4/go.mod h1:C5gboKD0TJPqWDTVTtrQNfRbiBwHZGo8UTqP/9/XvLI= github.com/vmihailenco/msgpack/v5 v5.3.5 h1:5gO0H1iULLWGhs2H5tbAHIZTV8/cYafcFOr9znI5mJU= github.com/vmihailenco/msgpack/v5 v5.3.5/go.mod h1:7xyJ9e+0+9SaZT0Wt1RGleJXzli6Q/V5KbhBonMG9jc= -github.com/vmihailenco/tagparser v0.1.2 h1:gnjoVuB/kljJ5wICEEOpx98oXMWPLj22G67Vbd1qPqc= -github.com/vmihailenco/tagparser v0.1.2/go.mod h1:OeAg3pn3UbLjkWt+rN9oFYB6u/cQgqMEUPoW2WPyhdI= github.com/vmihailenco/tagparser/v2 v2.0.0 h1:y09buUbR+b5aycVFQs/g70pqKVZNBmxwAhO7/IwNM9g= github.com/vmihailenco/tagparser/v2 v2.0.0/go.mod h1:Wri+At7QHww0WTrCBeu4J6bNtoV6mEfg5OIWRZA9qds= github.com/willf/bitset v1.1.3/go.mod h1:RjeCKbqT1RxIR/KWY6phxZiaY1IyutSBfGjNPySAYV4= -github.com/wsddn/go-ecdh v0.0.0-20161211032359-48726bab9208/go.mod h1:IotVbo4F+mw0EzQ08zFqg7pK3FebNXpaMsRy2RT+Ees= github.com/xlab/treeprint v0.0.0-20180616005107-d6fb6747feb6/go.mod h1:ce1O1j6UtZfjr22oyGxGLbauSBp2YVXpARAosm7dHBg= github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= github.com/ybbus/jsonrpc v2.1.2+incompatible/go.mod h1:XJrh1eMSzdIYFbM08flv0wp5G35eRniyeGut1z+LSiE= @@ -647,9 +577,9 @@ golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20200115085410-6d4e4cb37c7d/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200510223506-06a226fb4e37/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20200820211705-5c72a883971a h1:vclmkQCjlDX5OydZ9wv8rBCcS0QyQY66Mpf/7BZbInM= golang.org/x/crypto v0.0.0-20200820211705-5c72a883971a/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20201221181555-eec23a3978ad/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= +golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2 h1:It14KIkyBFYkHkwZ7k45minvA9aorojkyjGk9KJ5B/w= golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20180807140117-3d87b88a115f/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= @@ -657,7 +587,6 @@ golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL golang.org/x/exp v0.0.0-20190125153040-c74c464bbbf2/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= -golang.org/x/exp v0.0.0-20190731235908-ec7cb31e5a56/go.mod h1:JhuoJpWY28nO4Vef9tZUw9qufEGTyX1+7lmHxV5q5G4= golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek= golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY= golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= @@ -682,12 +611,10 @@ golang.org/x/lint v0.0.0-20201208152925-83fdc39ff7b5/go.mod h1:3xt1FjdF8hUf6vQPI golang.org/x/lint v0.0.0-20210508222113-6edffad5e616/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= -golang.org/x/mobile v0.0.0-20200801112145-973feb4309de/go.mod h1:skQtrUTUwhdJvXM/2KKJzY8pDgNr9I/FOMqDVRPBUS4= golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= -golang.org/x/mod v0.1.1-0.20191209134235-331c550502dd/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= @@ -697,7 +624,6 @@ golang.org/x/net v0.0.0-20180719180050-a680a1efc54d/go.mod h1:mL1N/T3taQHkDXs73r golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20181011144130-49bb7cea24b1/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181023162649-9b4f9f5ad519/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -728,7 +654,6 @@ golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81R golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20200813134508-3edf25e44fcc/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= -golang.org/x/net v0.0.0-20200904194848-62affa334b73/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201031054903-ff519b6c9102/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= @@ -737,7 +662,6 @@ golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v golang.org/x/net v0.0.0-20210220033124-5f55cee0dc0d/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210316092652-d523dce5a7f4/go.mod h1:RBQZq4jEuRlivfhVLdyRGr576XBO4/greRjx4P4O3yc= -golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4 h1:4nGaVu0QrbjT/AK2PRLuQfQuh6DJve+pELhqTdAj3x0= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d h1:20cMwl2fHAzkJMEA+8J4JgqBQcQGzbisXo31MIeenXI= golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= @@ -808,10 +732,8 @@ golang.org/x/sys v0.0.0-20200519105757-fe76b779f299/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200814200057-3d37ad5750ed/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200824131525-c12d262b63d8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200826173525-f9321e4c35a6/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200905004654-be1d3432aa8f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200909081042-eff7692f9009/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -829,7 +751,6 @@ golang.org/x/sys v0.0.0-20210403161142-5e06dd20ab57/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210420205809-ac73e9fd8988/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c h1:F1jZWGFhYfh0Ci55sIpILtKKK8p3i2/krTr0H1rg74I= golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210816183151-1e6c022a8912 h1:uCLL3g5wH2xjxVREVuAbP9JM5PPKjRbXKRa6IBjkzmU= golang.org/x/sys v0.0.0-20210816183151-1e6c022a8912/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -876,7 +797,6 @@ golang.org/x/tools v0.0.0-20191130070609-6e064ea0cf2d/go.mod h1:b+2E5dAYhXwXZwtn golang.org/x/tools v0.0.0-20191216173652-a0e659d51361/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200108203644-89082a384178/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200117012304-6edc0a871e69/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200117161641-43d50277825c/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200122220014-bf1340f18c4a/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= @@ -1023,8 +943,8 @@ google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQ gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= From d7369003cbfae6ced75aa9611ee768a8de6cbe27 Mon Sep 17 00:00:00 2001 From: Madhur Shrimal Date: Tue, 9 Nov 2021 14:58:11 -0800 Subject: [PATCH 280/376] add correct progress where genesis block isn't 0 (#246) * add correct progress where genesis block isn't 0 * lint * feedback --- pkg/results/data_results.go | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/pkg/results/data_results.go b/pkg/results/data_results.go index 4d1ef51f..e2aca1c5 100644 --- a/pkg/results/data_results.go +++ b/pkg/results/data_results.go @@ -306,6 +306,11 @@ func ComputeCheckDataProgress( } tipIndex := networkStatus.CurrentBlockIdentifier.Index + genesisBlockIndex := int64(0) + if networkStatus.GenesisBlockIdentifier != nil { + genesisBlockIndex = networkStatus.GenesisBlockIdentifier.Index + } + // Get current tip in the case that re-orgs occurred // or a custom start index was provied. headBlock, err := blockStorage.GetHeadBlockIdentifier(ctx) @@ -357,11 +362,14 @@ func ComputeCheckDataProgress( new(big.Float).SetInt(elapsedTime), ) blocksPerSecondFloat, _ := blocksPerSecond.Float64() + + // some blockchains don't start their genesis block from 0 height + // So take the height of genesis block and calculate sync percentage based on that blocksSynced := new( big.Float, ).Quo( - new(big.Float).SetInt64(headBlock.Index), - new(big.Float).SetInt64(tipIndex), + new(big.Float).SetInt64(headBlock.Index - genesisBlockIndex), + new(big.Float).SetInt64(tipIndex - genesisBlockIndex), ) blocksSyncedFloat, _ := blocksSynced.Float64() From 02d1dae0b927a64d68bbe883a2865f9743e108db Mon Sep 17 00:00:00 2001 From: Matheus Degiovani Date: Fri, 12 Nov 2021 18:31:16 -0300 Subject: [PATCH 281/376] Update Version (#254) The 'version' subcommand was still showing the old v0.6.7 version. --- cmd/root.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/root.go b/cmd/root.go index 130ee0d3..d60d68c3 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -294,6 +294,6 @@ var versionCmd = &cobra.Command{ Use: "version", Short: "Print rosetta-cli version", Run: func(cmd *cobra.Command, args []string) { - fmt.Println("v0.6.7") + fmt.Println("v0.7.3") }, } From d5ccb7c5cdfc60cd23ff7cb010dea1bf19f80547 Mon Sep 17 00:00:00 2001 From: Steven Yuan Date: Mon, 15 Nov 2021 14:40:23 -0800 Subject: [PATCH 282/376] Add m1 mac support to install script (#255) --- scripts/install.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/install.sh b/scripts/install.sh index 86ccad9b..669b8460 100755 --- a/scripts/install.sh +++ b/scripts/install.sh @@ -67,6 +67,7 @@ execute() { get_binaries() { case "$PLATFORM" in darwin/amd64) BINARY="rosetta-cli" ;; + darwin/arm64) BINARY="rosetta-cli" ;; linux/amd64) BINARY="rosetta-cli" ;; linux/arm64) BINARY="rosetta-cli" ;; linux/mips64) BINARY="rosetta-cli" ;; From 312b2de376eda0c4ff44a001d11fab2fdab983ba Mon Sep 17 00:00:00 2001 From: Madhur Shrimal Date: Thu, 18 Nov 2021 18:56:01 -0800 Subject: [PATCH 283/376] flag to print out error stack (#257) * flag to print out error stack * added config --- configuration/configuration_test.go | 19 ++++++++++--------- configuration/types.go | 4 ++++ examples/configuration/default.json | 1 + go.mod | 1 + pkg/results/construction_results.go | 10 ++++++++-- pkg/results/data_results.go | 14 ++++++++++---- 6 files changed, 34 insertions(+), 15 deletions(-) diff --git a/configuration/configuration_test.go b/configuration/configuration_test.go index bdc91253..9328be08 100644 --- a/configuration/configuration_test.go +++ b/configuration/configuration_test.go @@ -62,15 +62,16 @@ var ( Blockchain: "sweet", Network: "sweeter", }, - OnlineURL: "http://hasudhasjkdk", - MaxOnlineConnections: 10, - HTTPTimeout: 21, - MaxRetries: 1000, - MaxSyncConcurrency: 12, - TipDelay: 1231, - MaxReorgDepth: 12, - SeenBlockWorkers: 300, - SerialBlockWorkers: 200, + OnlineURL: "http://hasudhasjkdk", + MaxOnlineConnections: 10, + HTTPTimeout: 21, + MaxRetries: 1000, + MaxSyncConcurrency: 12, + TipDelay: 1231, + MaxReorgDepth: 12, + SeenBlockWorkers: 300, + SerialBlockWorkers: 200, + ErrorStackTraceEnabled: true, Construction: &ConstructionConfiguration{ OfflineURL: "https://ashdjaksdkjshdk", MaxOfflineConnections: 21, diff --git a/configuration/types.go b/configuration/types.go index 817608aa..95cc6209 100644 --- a/configuration/types.go +++ b/configuration/types.go @@ -421,6 +421,10 @@ type Configuration struct { // specific validation will be done ValidationFile string `json:"validation_file,omitempty"` + // ErrorStackTraceEnabled if true then it will print error stack trace + // if the data or construction check fails + ErrorStackTraceEnabled bool `json:"error_stack_trace_enabled"` + Construction *ConstructionConfiguration `json:"construction"` Data *DataConfiguration `json:"data"` } diff --git a/examples/configuration/default.json b/examples/configuration/default.json index 1d5a448a..91dabf3f 100644 --- a/examples/configuration/default.json +++ b/examples/configuration/default.json @@ -15,6 +15,7 @@ "log_configuration": false, "compression_disabled": false, "memory_limit_disabled": false, + "error_stack_trace_enabled": false, "construction": null, "data": { "active_reconciliation_concurrency": 16, diff --git a/go.mod b/go.mod index 17f98260..25fcab95 100644 --- a/go.mod +++ b/go.mod @@ -6,6 +6,7 @@ require ( github.com/coinbase/rosetta-sdk-go v0.7.1 github.com/fatih/color v1.13.0 github.com/olekukonko/tablewriter v0.0.5 + github.com/pkg/errors v0.9.1 github.com/spf13/cobra v1.2.1 github.com/stretchr/testify v1.7.0 golang.org/x/sync v0.0.0-20210220032951-036812b2e83c diff --git a/pkg/results/construction_results.go b/pkg/results/construction_results.go index 22f8b657..82957c3f 100644 --- a/pkg/results/construction_results.go +++ b/pkg/results/construction_results.go @@ -21,6 +21,8 @@ import ( "os" "strconv" + pkgError "github.com/pkg/errors" + "github.com/coinbase/rosetta-cli/configuration" "github.com/coinbase/rosetta-sdk-go/storage/modules" @@ -83,7 +85,7 @@ func ComputeCheckConstructionResults( } if err != nil { - results.Error = err.Error() + results.Error = fmt.Sprintf("%+v", err) // We never want to populate an end condition // if there was an error! @@ -291,7 +293,7 @@ func FetchCheckConstructionStatus(url string) (*CheckConstructionStatus, error) return &status, nil } -// ExitConstruction exits check:data, logs the test results to the console, +// ExitConstruction exits check:construction, logs the test results to the console, // and to a provided output path. func ExitConstruction( config *configuration.Configuration, @@ -299,6 +301,10 @@ func ExitConstruction( jobStorage *modules.JobStorage, err error, ) error { + if config.ErrorStackTraceEnabled { + err = pkgError.WithStack(err) + } + results := ComputeCheckConstructionResults( config, err, diff --git a/pkg/results/data_results.go b/pkg/results/data_results.go index e2aca1c5..b765174f 100644 --- a/pkg/results/data_results.go +++ b/pkg/results/data_results.go @@ -23,6 +23,8 @@ import ( "os" "strconv" + pkgError "github.com/pkg/errors" + "github.com/coinbase/rosetta-cli/configuration" "github.com/coinbase/rosetta-sdk-go/asserter" @@ -308,7 +310,7 @@ func ComputeCheckDataProgress( genesisBlockIndex := int64(0) if networkStatus.GenesisBlockIdentifier != nil { - genesisBlockIndex = networkStatus.GenesisBlockIdentifier.Index + genesisBlockIndex = networkStatus.GenesisBlockIdentifier.Index } // Get current tip in the case that re-orgs occurred @@ -368,8 +370,8 @@ func ComputeCheckDataProgress( blocksSynced := new( big.Float, ).Quo( - new(big.Float).SetInt64(headBlock.Index - genesisBlockIndex), - new(big.Float).SetInt64(tipIndex - genesisBlockIndex), + new(big.Float).SetInt64(headBlock.Index-genesisBlockIndex), + new(big.Float).SetInt64(tipIndex-genesisBlockIndex), ) blocksSyncedFloat, _ := blocksSynced.Float64() @@ -669,7 +671,7 @@ func ComputeCheckDataResults( } if err != nil { - results.Error = err.Error() + results.Error = fmt.Sprintf("%+v", err) // If all tests pass, but we still encountered an error, // then we hard exit without showing check:data results @@ -707,6 +709,10 @@ func ExitData( endCondition configuration.CheckDataEndCondition, endConditionDetail string, ) error { + if config.ErrorStackTraceEnabled { + err = pkgError.WithStack(err) + } + results := ComputeCheckDataResults( config, err, From 053ab62ed7a9b211cf1efad2c33938dbaf79336a Mon Sep 17 00:00:00 2001 From: racbc Date: Mon, 22 Nov 2021 13:08:11 -0600 Subject: [PATCH 284/376] Fix typos in types.go (#253) * Fix typos in types.go Fixed misspelling of the word `discrepancy` in the `InactiveDiscrepancySearchDisabled` type. * Fix misspelling of "discrepancy" in coinbase/rosetta-cli files Fixed the misspelling in several files. * Add json generated file --- configuration/types.go | 8 ++++---- examples/configuration/default.json | 2 +- examples/configuration/simple.json | 2 +- pkg/tester/data.go | 4 ++-- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/configuration/types.go b/configuration/types.go index 95cc6209..1aa0a991 100644 --- a/configuration/types.go +++ b/configuration/types.go @@ -278,11 +278,11 @@ type DataConfiguration struct { // been drained (if reconciliation is enabled). ReconciliationDrainDisabled bool `json:"reconciliation_drain_disabled"` - // InactiveDiscrepencySearchDisabled is a boolean indicating if a search - // should be performed to find any inactive reconciliation discrepencies. + // InactiveDiscrepancySearchDisabled is a boolean indicating if a search + // should be performed to find any inactive reconciliation discrepancies. // Note, a search will never be performed if historical balance lookup // is disabled. - InactiveDiscrepencySearchDisabled bool `json:"inactive_discrepency_search_disabled"` + InactiveDiscrepancySearchDisabled bool `json:"inactive_discrepancy_search_disabled"` // BalanceTrackingDisabled is a boolean that indicates balances calculation // should not be attempted. When first testing an implemenation, it can be @@ -301,7 +301,7 @@ type DataConfiguration struct { // If no blocks have ever been synced, syncing will start from genesis. StartIndex *int64 `json:"start_index,omitempty"` - // EndCondition contains the conditions for the syncer to stop + // EndCondition contains the conditions for the syncer to stop. EndConditions *DataEndConditions `json:"end_conditions,omitempty"` // StatusPort allows the caller to query a running check:data diff --git a/examples/configuration/default.json b/examples/configuration/default.json index 91dabf3f..33792cdf 100644 --- a/examples/configuration/default.json +++ b/examples/configuration/default.json @@ -31,7 +31,7 @@ "interesting_accounts": "", "reconciliation_disabled": false, "reconciliation_drain_disabled": false, - "inactive_discrepency_search_disabled": false, + "inactive_discrepancy_search_disabled": false, "balance_tracking_disabled": false, "coin_tracking_disabled": false, "status_port": 9090, diff --git a/examples/configuration/simple.json b/examples/configuration/simple.json index cbf87552..adbbd21b 100644 --- a/examples/configuration/simple.json +++ b/examples/configuration/simple.json @@ -10,7 +10,7 @@ "data": { "historical_balance_enabled": false, "reconciliation_disabled": true, - "inactive_discrepency_search_disabled": true, + "inactive_discrepancy_search_disabled": true, "balance_tracking_disabled": true, "end_conditions": { "tip": true diff --git a/pkg/tester/data.go b/pkg/tester/data.go index 336664cc..1ca4ed84 100644 --- a/pkg/tester/data.go +++ b/pkg/tester/data.go @@ -931,8 +931,8 @@ func (t *DataTester) HandleErr(err error, sigListeners *[]context.CancelFunc) er ) } - if t.config.Data.InactiveDiscrepencySearchDisabled { - color.Yellow("Search for inactive reconciliation discrepency is disabled") + if t.config.Data.InactiveDiscrepancySearchDisabled { + color.Yellow("Search for inactive reconciliation discrepancy is disabled") return results.ExitData( t.config, t.counterStorage, From 88a2e77131df8b009b90a289d88a7b92c335e590 Mon Sep 17 00:00:00 2001 From: Madhur Shrimal Date: Wed, 8 Dec 2021 11:12:22 -0800 Subject: [PATCH 285/376] [chore] update to rosetta-sdk-go@v0.7.2 (#262) * update to rosetta-sdk-go@v0.7.2 * go mod tidy --- go.mod | 2 +- go.sum | 21 ++++++++++----------- 2 files changed, 11 insertions(+), 12 deletions(-) diff --git a/go.mod b/go.mod index 25fcab95..fe0c215b 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/coinbase/rosetta-cli go 1.16 require ( - github.com/coinbase/rosetta-sdk-go v0.7.1 + github.com/coinbase/rosetta-sdk-go v0.7.2 github.com/fatih/color v1.13.0 github.com/olekukonko/tablewriter v0.0.5 github.com/pkg/errors v0.9.1 diff --git a/go.sum b/go.sum index dc98dc64..355acd2f 100644 --- a/go.sum +++ b/go.sum @@ -57,8 +57,8 @@ github.com/Azure/go-autorest/tracing v0.5.0/go.mod h1:r/s2XiOKccPW3HrqB+W0TQzfbt github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/DATA-DOG/go-sqlmock v1.3.3/go.mod h1:f/Ixk793poVmq4qj/V1dPUg2JEAKC73Q5eFN3EC/SaM= -github.com/DataDog/zstd v1.4.8 h1:Rpmta4xZ/MgZnriKNd24iZMhGpP5dvUcs/uqfBapKZY= -github.com/DataDog/zstd v1.4.8/go.mod h1:g4AWEaM3yOg3HYfnJ3YIawPnVdXJh9QME85blwSAmyw= +github.com/DataDog/zstd v1.5.0 h1:+K/VEwIAaPcHiMtQvpLD4lqW7f0Gk3xdYZmI1hD+CXo= +github.com/DataDog/zstd v1.5.0/go.mod h1:g4AWEaM3yOg3HYfnJ3YIawPnVdXJh9QME85blwSAmyw= github.com/OneOfOne/xxhash v1.2.2 h1:KMrpdQIwFcEqXDklaen+P1axHaj9BSKzvpUUfnHldSE= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= github.com/StackExchange/wmi v0.0.0-20180116203802-5d049714c4a6/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg= @@ -123,8 +123,8 @@ github.com/cloudflare/cloudflare-go v0.14.0/go.mod h1:EnwdgGMaFOruiPZRFSgn+TsQ3h github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= -github.com/coinbase/rosetta-sdk-go v0.7.1 h1:J1erjPiDnyDIHwscxXSrx9GnnFhzb8zDhx/On2j7JF8= -github.com/coinbase/rosetta-sdk-go v0.7.1/go.mod h1:MZX7tpDNCZOHm1WpydIfNwpHUDXJR1Pt4xeuABqfvQo= +github.com/coinbase/rosetta-sdk-go v0.7.2 h1:uCNrASIyt7rV9bA3gzPG3JDlxVP5v/zLgi01GWngncM= +github.com/coinbase/rosetta-sdk-go v0.7.2/go.mod h1:wk9dvjZFSZiWSNkFuj3dMleTA1adLFotg5y71PhqKB4= github.com/consensys/bavard v0.1.8-0.20210406032232-f3452dc9b572/go.mod h1:Bpd0/3mZuaj6Sj+PqrmIquiOKy397AKGThQPaGzNXAQ= github.com/consensys/gnark-crypto v0.4.1-0.20210426202927-39ac3d4b3f1f/go.mod h1:815PAHg3wvysy0SyIqanF8gZ0Y1wjk/hrDHD/iT88+Q= github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= @@ -172,10 +172,9 @@ github.com/envoyproxy/go-control-plane v0.9.7/go.mod h1:cwu0lG7PUMfa9snN8LXBig5y github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= github.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= -github.com/ethereum/go-ethereum v1.10.11 h1:KKIcwpmur9iTaVbR2dxlHu+peHVhU+/KX//NWvT1n9U= -github.com/ethereum/go-ethereum v1.10.11/go.mod h1:W3yfrFyL9C1pHcwY5hmRHVDaorTiQxhYBkKyu5mEDHw= +github.com/ethereum/go-ethereum v1.10.13 h1:DEYFP9zk+Gruf3ae1JOJVhNmxK28ee+sMELPLgYTXpA= +github.com/ethereum/go-ethereum v1.10.13/go.mod h1:W3yfrFyL9C1pHcwY5hmRHVDaorTiQxhYBkKyu5mEDHw= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= -github.com/fatih/color v1.10.0/go.mod h1:ELkj/draVOlAH/xkhN6mQ50Qd0MPOk5AAr3maGEBuJM= github.com/fatih/color v1.13.0 h1:8LOYc1KYPPmyKMuN8QV2DNRWNbLo6LZ0iLs8+mlH53w= github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk= github.com/fjl/memsize v0.0.0-20190710130421-bcb5799ab5e5/go.mod h1:VvhXpOYNQvB+uIk2RvXzuaQtkQJzzIx6lSBe1xv7hi0= @@ -408,8 +407,8 @@ github.com/mitchellh/iochan v1.0.0/go.mod h1:JwYml1nuB7xOzsp52dPpHFffvOCDupsG0Qu github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/mitchellh/mapstructure v1.4.1/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= -github.com/mitchellh/mapstructure v1.4.2 h1:6h7AQ0yhTcIsmFmnAwQls75jp2Gzs4iB8W7pjMO+rqo= -github.com/mitchellh/mapstructure v1.4.2/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= +github.com/mitchellh/mapstructure v1.4.3 h1:OVowDSCllw/YjdLkam3/sm7wEtOy59d8ndGgCcyj8cs= +github.com/mitchellh/mapstructure v1.4.3/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/mitchellh/pointerstructure v1.2.0/go.mod h1:BRAsLI5zgXmw97Lf6s25bs8ohIXc3tViBH44KcwB2g4= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= @@ -516,8 +515,8 @@ github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7/go.mod h1:q4W45IWZaF22tdD+VEXcAWRA037jwmWEB5VWYORlTpc= github.com/tidwall/gjson v1.10.2/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= -github.com/tidwall/gjson v1.11.0 h1:C16pk7tQNiH6VlCrtIXL1w8GaOsi1X3W8KDkE1BuYd4= -github.com/tidwall/gjson v1.11.0/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= +github.com/tidwall/gjson v1.12.0 h1:61wEp/qfvFnqKH/WCI3M8HuRut+mHT6Mr82QrFmM2SY= +github.com/tidwall/gjson v1.12.0/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA= github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM= github.com/tidwall/pretty v1.2.0 h1:RWIZEg2iJ8/g6fDDYzMpobmaoGh5OLl4AXtGUGPcqCs= From 561f050bb2ae34f734c23ec75baaa37937a70179 Mon Sep 17 00:00:00 2001 From: Madhur Shrimal Date: Thu, 16 Dec 2021 10:49:22 -0800 Subject: [PATCH 286/376] config consistency (#266) * config consistency * change commit --- configuration/configuration_test.go | 36 ++++++++++++++--------------- configuration/types.go | 12 +++++----- examples/configuration/default.json | 2 +- examples/configuration/simple.json | 2 +- pkg/results/construction_results.go | 2 +- pkg/results/data_results.go | 2 +- pkg/tester/data.go | 4 ++-- 7 files changed, 30 insertions(+), 30 deletions(-) diff --git a/configuration/configuration_test.go b/configuration/configuration_test.go index 9328be08..ecf88b49 100644 --- a/configuration/configuration_test.go +++ b/configuration/configuration_test.go @@ -29,13 +29,13 @@ import ( ) var ( - startIndex = int64(89) - badStartIndex = int64(-10) - goodCoverage = float64(0.33) - badCoverage = float64(-2) - endTip = false - historicalEnabled = true - fakeWorkflows = []*job.Workflow{ + startIndex = int64(89) + badStartIndex = int64(-10) + goodCoverage = float64(0.33) + badCoverage = float64(-2) + endTip = false + historicalDisabled = false + fakeWorkflows = []*job.Workflow{ { Name: string(job.CreateAccount), Concurrency: job.ReservedWorkflowConcurrency, @@ -62,16 +62,16 @@ var ( Blockchain: "sweet", Network: "sweeter", }, - OnlineURL: "http://hasudhasjkdk", - MaxOnlineConnections: 10, - HTTPTimeout: 21, - MaxRetries: 1000, - MaxSyncConcurrency: 12, - TipDelay: 1231, - MaxReorgDepth: 12, - SeenBlockWorkers: 300, - SerialBlockWorkers: 200, - ErrorStackTraceEnabled: true, + OnlineURL: "http://hasudhasjkdk", + MaxOnlineConnections: 10, + HTTPTimeout: 21, + MaxRetries: 1000, + MaxSyncConcurrency: 12, + TipDelay: 1231, + MaxReorgDepth: 12, + SeenBlockWorkers: 300, + SerialBlockWorkers: 200, + ErrorStackTraceDisabled: false, Construction: &ConstructionConfiguration{ OfflineURL: "https://ashdjaksdkjshdk", MaxOfflineConnections: 21, @@ -92,7 +92,7 @@ var ( InactiveReconciliationConcurrency: 2938, InactiveReconciliationFrequency: 3, ReconciliationDisabled: false, - HistoricalBalanceEnabled: &historicalEnabled, + HistoricalBalanceDisabled: &historicalDisabled, StartIndex: &startIndex, StatusPort: 123, EndConditions: &DataEndConditions{ diff --git a/configuration/types.go b/configuration/types.go index 1aa0a991..c0f53c3a 100644 --- a/configuration/types.go +++ b/configuration/types.go @@ -258,11 +258,11 @@ type DataConfiguration struct { // beginning syncing, it will be ignored. BootstrapBalances string `json:"bootstrap_balances"` - // HistoricalBalanceEnabled is a boolean that dictates how balance lookup is performed. - // When set to true, balances are looked up at the block where a balance + // HistoricalBalanceDisabled is a boolean that dictates how balance lookup is performed. + // When set to false, balances are looked up at the block where a balance // change occurred instead of at the current block. Blockchains that do not support - // historical balance lookup should set this to false. - HistoricalBalanceEnabled *bool `json:"historical_balance_enabled,omitempty"` + // historical balance lookup should set this to true. + HistoricalBalanceDisabled *bool `json:"historical_balance_disabled,omitempty"` // InterestingAccounts is a path to a file listing all accounts to check on each block. Look // at the examples directory for an example of how to structure this file. @@ -421,9 +421,9 @@ type Configuration struct { // specific validation will be done ValidationFile string `json:"validation_file,omitempty"` - // ErrorStackTraceEnabled if true then it will print error stack trace + // ErrorStackTraceDisabled if false then it will print error stack trace // if the data or construction check fails - ErrorStackTraceEnabled bool `json:"error_stack_trace_enabled"` + ErrorStackTraceDisabled bool `json:"error_stack_trace_disabled"` Construction *ConstructionConfiguration `json:"construction"` Data *DataConfiguration `json:"data"` diff --git a/examples/configuration/default.json b/examples/configuration/default.json index 33792cdf..e119f906 100644 --- a/examples/configuration/default.json +++ b/examples/configuration/default.json @@ -15,7 +15,7 @@ "log_configuration": false, "compression_disabled": false, "memory_limit_disabled": false, - "error_stack_trace_enabled": false, + "error_stack_trace_disabled": false, "construction": null, "data": { "active_reconciliation_concurrency": 16, diff --git a/examples/configuration/simple.json b/examples/configuration/simple.json index adbbd21b..95e9479a 100644 --- a/examples/configuration/simple.json +++ b/examples/configuration/simple.json @@ -8,7 +8,7 @@ "http_timeout": 10, "tip_delay": 300, "data": { - "historical_balance_enabled": false, + "historical_balance_disabled": true, "reconciliation_disabled": true, "inactive_discrepancy_search_disabled": true, "balance_tracking_disabled": true, diff --git a/pkg/results/construction_results.go b/pkg/results/construction_results.go index 82957c3f..32915338 100644 --- a/pkg/results/construction_results.go +++ b/pkg/results/construction_results.go @@ -301,7 +301,7 @@ func ExitConstruction( jobStorage *modules.JobStorage, err error, ) error { - if config.ErrorStackTraceEnabled { + if !config.ErrorStackTraceDisabled { err = pkgError.WithStack(err) } diff --git a/pkg/results/data_results.go b/pkg/results/data_results.go index b765174f..e6674094 100644 --- a/pkg/results/data_results.go +++ b/pkg/results/data_results.go @@ -709,7 +709,7 @@ func ExitData( endCondition configuration.CheckDataEndCondition, endConditionDetail string, ) error { - if config.ErrorStackTraceEnabled { + if !config.ErrorStackTraceDisabled { err = pkgError.WithStack(err) } diff --git a/pkg/tester/data.go b/pkg/tester/data.go index 1ca4ed84..a24c67ed 100644 --- a/pkg/tester/data.go +++ b/pkg/tester/data.go @@ -230,8 +230,8 @@ func InitializeData( // Determine if we should perform historical balance lookups var historicalBalanceEnabled bool - if config.Data.HistoricalBalanceEnabled != nil { - historicalBalanceEnabled = *config.Data.HistoricalBalanceEnabled + if config.Data.HistoricalBalanceDisabled != nil { + historicalBalanceEnabled = !*config.Data.HistoricalBalanceDisabled } else { // we must look it up historicalBalanceEnabled = networkOptions.Allow.HistoricalBalanceLookup } From 36b2bf9b3a538c82d750c1ae42b3553888fe9ec0 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 4 Jan 2022 10:02:42 -0800 Subject: [PATCH 287/376] Bump github.com/spf13/cobra from 1.2.1 to 1.3.0 (#265) Bumps [github.com/spf13/cobra](https://github.com/spf13/cobra) from 1.2.1 to 1.3.0. - [Release notes](https://github.com/spf13/cobra/releases) - [Changelog](https://github.com/spf13/cobra/blob/master/CHANGELOG.md) - [Commits](https://github.com/spf13/cobra/compare/v1.2.1...v1.3.0) --- updated-dependencies: - dependency-name: github.com/spf13/cobra dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 214 ++++++++++++++++++++++++++++++++++++++++++++++++--------- 2 files changed, 182 insertions(+), 34 deletions(-) diff --git a/go.mod b/go.mod index fe0c215b..6308774e 100644 --- a/go.mod +++ b/go.mod @@ -7,7 +7,7 @@ require ( github.com/fatih/color v1.13.0 github.com/olekukonko/tablewriter v0.0.5 github.com/pkg/errors v0.9.1 - github.com/spf13/cobra v1.2.1 + github.com/spf13/cobra v1.3.0 github.com/stretchr/testify v1.7.0 golang.org/x/sync v0.0.0-20210220032951-036812b2e83c ) diff --git a/go.sum b/go.sum index 355acd2f..718bbfe4 100644 --- a/go.sum +++ b/go.sum @@ -20,6 +20,15 @@ cloud.google.com/go v0.74.0/go.mod h1:VV1xSbzvo+9QJOxLDaJfTjx5e+MePCpCWwvftOeQmW cloud.google.com/go v0.78.0/go.mod h1:QjdrLG0uq+YwhjoVOLsS1t7TW8fs36kLs4XO5R5ECHg= cloud.google.com/go v0.79.0/go.mod h1:3bzgcEeQlzbuEAYu4mrWhKqWjmpprinYgKJLgKHnbb8= cloud.google.com/go v0.81.0/go.mod h1:mk/AM35KwGk/Nm2YSeZbxXdrNK3KZOYHmLkOqC2V6E0= +cloud.google.com/go v0.83.0/go.mod h1:Z7MJUsANfY0pYPdw0lbnivPx4/vhy/e2FEkSkF7vAVY= +cloud.google.com/go v0.84.0/go.mod h1:RazrYuxIK6Kb7YrzzhPoLmCVzl7Sup4NrbKPg8KHSUM= +cloud.google.com/go v0.87.0/go.mod h1:TpDYlFy7vuLzZMMZ+B6iRiELaY7z/gJPaqbMx6mlWcY= +cloud.google.com/go v0.90.0/go.mod h1:kRX0mNRHe0e2rC6oNakvwQqzyDmg57xJ+SZU1eT2aDQ= +cloud.google.com/go v0.93.3/go.mod h1:8utlLll2EF5XMAV15woO4lSbWQlk8rer9aLOfLh7+YI= +cloud.google.com/go v0.94.1/go.mod h1:qAlAugsXlC+JWO+Bke5vCtc9ONxjQT3drlTTnAplMW4= +cloud.google.com/go v0.97.0/go.mod h1:GF7l59pYBVlXQIBLx3a761cZ41F9bBH3JUlihCt2Udc= +cloud.google.com/go v0.98.0/go.mod h1:ua6Ush4NALrHk5QXDWnjvZHN93OuF0HfuEPq9I1X0cM= +cloud.google.com/go v0.99.0/go.mod h1:w0Xx2nLzqWJPuozYQX+hFfCSI8WioryfRDzkoI/Y2ZA= cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc= @@ -29,7 +38,7 @@ cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM7 cloud.google.com/go/bigtable v1.2.0/go.mod h1:JcVAOl45lrTmQfLj7T6TxyMzIN/3FGGcFm+2xVAli2o= cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= -cloud.google.com/go/firestore v1.1.0/go.mod h1:ulACoGHTpvq5r8rxGJ4ddJZBZqakUQqClKRT5SZwBmk= +cloud.google.com/go/firestore v1.6.1/go.mod h1:asNXNOzBdyVQmEU+ggO8UPodTkEVFW5Qx+rwHnAz+EY= cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA= @@ -57,6 +66,7 @@ github.com/Azure/go-autorest/tracing v0.5.0/go.mod h1:r/s2XiOKccPW3HrqB+W0TQzfbt github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/DATA-DOG/go-sqlmock v1.3.3/go.mod h1:f/Ixk793poVmq4qj/V1dPUg2JEAKC73Q5eFN3EC/SaM= +github.com/DataDog/datadog-go v3.2.0+incompatible/go.mod h1:LButxg5PwREeZtORoXG3tL4fMGNddJ+vMq1mwgfaqoQ= github.com/DataDog/zstd v1.5.0 h1:+K/VEwIAaPcHiMtQvpLD4lqW7f0Gk3xdYZmI1hD+CXo= github.com/DataDog/zstd v1.5.0/go.mod h1:g4AWEaM3yOg3HYfnJ3YIawPnVdXJh9QME85blwSAmyw= github.com/OneOfOne/xxhash v1.2.2 h1:KMrpdQIwFcEqXDklaen+P1axHaj9BSKzvpUUfnHldSE= @@ -68,7 +78,9 @@ github.com/Zilliqa/gozilliqa-sdk v1.2.1-0.20201201074141-dd0ecada1be6/go.mod h1: github.com/aead/siphash v1.0.1/go.mod h1:Nywa3cDsYNNK3gaciGTWPwHt0wlpNV15vwmswBAUSII= github.com/ajstarks/svgo v0.0.0-20180226025133-644b8db467af/go.mod h1:K08gAheRH3/J6wwsYMMT4xOr94bZjxIelGM0+d/wbFw= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= +github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= +github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/allegro/bigcache v1.2.1-0.20190218064605-e24eb225f156/go.mod h1:Cb/ax3seSYIx7SuZdm2G2xzfwmv3TPSk2ucNfQESPXM= github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883/go.mod h1:rCTlJbsFo29Kk6CurOXKm700vrz8f0KW0JNfpkRJY/8= github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= @@ -76,7 +88,9 @@ github.com/apache/arrow/go/arrow v0.0.0-20191024131854-af6fa24be0db/go.mod h1:VT github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= +github.com/armon/go-metrics v0.3.10/go.mod h1:4O98XIr/9W0sxpJ8UaYkvjk10Iff7SnFrb4QAOwNTFc= github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= +github.com/armon/go-radix v1.0.0/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= github.com/aws/aws-sdk-go-v2 v1.2.0/go.mod h1:zEQs02YRBw1DjK0PoJv3ygDYOFTre1ejlJWl8FwAuQo= github.com/aws/aws-sdk-go-v2/config v1.1.1/go.mod h1:0XsVy9lBI/BCXm+2Tuvt39YmdHwS5unDQmxZOYe8F5Y= github.com/aws/aws-sdk-go-v2/credentials v1.1.1/go.mod h1:mM2iIjwl7LULWtS6JCACyInboHirisUUdkBPoTHMOUo= @@ -88,8 +102,8 @@ github.com/aws/aws-sdk-go-v2/service/sts v1.1.1/go.mod h1:Wi0EBZwiz/K44YliU0EKxq github.com/aws/smithy-go v1.1.0/go.mod h1:EzMw8dbp/YJL4A5/sbhGddag+NPT7q084agLbB9LgIw= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= +github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= -github.com/bketelsen/crypt v0.0.4/go.mod h1:aI6NrJ0pMGgvZKL1iVgXLnfIFJtfV+bKCoqOes/6LfM= github.com/bmizerany/pat v0.0.0-20170815010413-6226ea591a40/go.mod h1:8rLXio+WjiTceGBHIoTvn60HIbs7Hm7bcHjyrSqYB9c= github.com/boltdb/bolt v1.3.1/go.mod h1:clJnj/oiGkjum5o1McbSZDSLxVThjynRyGBgiAx27Ps= github.com/btcsuite/btcd v0.0.0-20190315201642-aa6e0f35703c/go.mod h1:DrZx5ec/dmnfpw9KyYoQyYo7d0KEvTkk/5M/vbZjAr8= @@ -111,18 +125,29 @@ github.com/c-bata/go-prompt v0.2.2/go.mod h1:VzqtzE2ksDBcdln8G7mk2RX9QyGjH+OVqOC github.com/cenkalti/backoff v2.2.1+incompatible h1:tNowT99t7UNflLxfYYSlKYsBpXdEet03Pg2g16Swow4= github.com/cenkalti/backoff v2.2.1+incompatible/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= +github.com/census-instrumentation/opencensus-proto v0.3.0/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/cp v0.1.0/go.mod h1:SOGHArjBr4JWaSDEVpWpo/hNg6RoKrls6Oh40hiwW+s= github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= +github.com/circonus-labs/circonus-gometrics v2.3.1+incompatible/go.mod h1:nmEj6Dob7S7YxXgwXpfOuvO54S+tGdZdw9fuRZt25Ag= +github.com/circonus-labs/circonusllhist v0.1.3/go.mod h1:kMXHVDlOchFAehlya5ePtbp5jckzBHf4XRpQvBOLI+I= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cloudflare/cloudflare-go v0.14.0/go.mod h1:EnwdgGMaFOruiPZRFSgn+TsQ3hQ7C/YWzIGLeu5c304= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= +github.com/cncf/udpa/go v0.0.0-20210930031921-04548b0d99d4/go.mod h1:6pvJx4me5XPnfI9Z40ddWsdw2W/uZgQLFXToKeRcDiI= +github.com/cncf/xds/go v0.0.0-20210312221358-fbca930ec8ed/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/cncf/xds/go v0.0.0-20210805033703-aa0b78936158/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/cncf/xds/go v0.0.0-20210922020428-25de7278fc84/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/cncf/xds/go v0.0.0-20211001041855-01bcc9b48dfe/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/cncf/xds/go v0.0.0-20211130200136-a8f946100490/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/coinbase/rosetta-sdk-go v0.7.2 h1:uCNrASIyt7rV9bA3gzPG3JDlxVP5v/zLgi01GWngncM= github.com/coinbase/rosetta-sdk-go v0.7.2/go.mod h1:wk9dvjZFSZiWSNkFuj3dMleTA1adLFotg5y71PhqKB4= github.com/consensys/bavard v0.1.8-0.20210406032232-f3452dc9b572/go.mod h1:Bpd0/3mZuaj6Sj+PqrmIquiOKy397AKGThQPaGzNXAQ= @@ -134,7 +159,7 @@ github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3Ee github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE= github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= -github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= +github.com/cpuguy83/go-md2man/v2 v2.0.1/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/cyberdelia/templates v0.0.0-20141128023046-ca7fffd4298c/go.mod h1:GyV+0YP4qX0UQ7r2MoYZ+AvYDp12OF5yg4q8rGnyNh4= github.com/dave/jennifer v1.2.0/go.mod h1:fIb+770HOpJ2fmN9EPPKOqm1vMGhB+TwXKMZhrIygKg= @@ -171,16 +196,22 @@ github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1m github.com/envoyproxy/go-control-plane v0.9.7/go.mod h1:cwu0lG7PUMfa9snN8LXBig5ynNVH9qI8YYLbd1fK2po= github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= github.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= +github.com/envoyproxy/go-control-plane v0.9.9-0.20210512163311-63b5d3c536b0/go.mod h1:hliV/p42l8fGbc6Y9bQ70uLwIvmJyVE5k4iMKlh8wCQ= +github.com/envoyproxy/go-control-plane v0.9.10-0.20210907150352-cf90f659a021/go.mod h1:AFq3mo9L8Lqqiid3OhADV3RfLJnjiw63cSpi+fDTRC0= +github.com/envoyproxy/go-control-plane v0.10.1/go.mod h1:AY7fTTXNdv/aJ2O5jwpxAPOWUZ7hQAEvzN5Pf27BkQQ= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= +github.com/envoyproxy/protoc-gen-validate v0.6.2/go.mod h1:2t7qjJNvHPx8IjnBOzl9E9/baC+qXE/TeeyBRzgJDws= github.com/ethereum/go-ethereum v1.10.13 h1:DEYFP9zk+Gruf3ae1JOJVhNmxK28ee+sMELPLgYTXpA= github.com/ethereum/go-ethereum v1.10.13/go.mod h1:W3yfrFyL9C1pHcwY5hmRHVDaorTiQxhYBkKyu5mEDHw= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= +github.com/fatih/color v1.9.0/go.mod h1:eQcE1qtQxscV5RaZvpXrrb8Drkc3/DdQ+uUYCNjL+zU= github.com/fatih/color v1.13.0 h1:8LOYc1KYPPmyKMuN8QV2DNRWNbLo6LZ0iLs8+mlH53w= github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk= github.com/fjl/memsize v0.0.0-20190710130421-bcb5799ab5e5/go.mod h1:VvhXpOYNQvB+uIk2RvXzuaQtkQJzzIx6lSBe1xv7hi0= github.com/fogleman/gg v1.2.1-0.20190220221249-0403632d5b90/go.mod h1:R/bRT+9gY/C5z7JzPU0zXsXHKM4/ayA+zqcVNZzPa1k= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= +github.com/fsnotify/fsnotify v1.5.1/go.mod h1:T3375wBYaZdLLcVNkcVbzGHY7f1l/uK5T5Ai1i3InKU= github.com/gballet/go-libpcsclite v0.0.0-20190607065134-2772fd86a8ff/go.mod h1:x7DCsMOv1taUwEWCzT4cmDeAkigA5/QCwUodaVOe8Ww= github.com/getkin/kin-openapi v0.53.0/go.mod h1:7Yn5whZr5kJi6t+kShccXS8ae1APpYTW6yheSwk8Yi4= github.com/getkin/kin-openapi v0.61.0/go.mod h1:7Yn5whZr5kJi6t+kShccXS8ae1APpYTW6yheSwk8Yi4= @@ -192,6 +223,7 @@ github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9 github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= +github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= github.com/go-ole/go-ole v1.2.1/go.mod h1:7FAglXiTm7HKlQRDeOQ6ZNUHidzCWXuZWq/1dTyBNF8= @@ -211,6 +243,7 @@ github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfU github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= @@ -219,6 +252,7 @@ github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4= github.com/golang/mock v1.5.0/go.mod h1:CWnOUgYIOo4TcNZ0wHX3YZCqsaM1I1Jvs6v3mP3KVu8= +github.com/golang/mock v1.6.0/go.mod h1:p6yTPP+5HYm5mzsMV8JkE6ZKdX+/wYM6Hr+LicevLPs= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= @@ -255,13 +289,15 @@ github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.6 h1:BKbKCqvP6I+rmFHt06ZmyQtvB8xAkWdhFyr0ZUNZcxQ= +github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/gofuzz v1.1.1-0.20200604201612-c04b05f3adfa/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= github.com/google/martian/v3 v3.1.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= +github.com/google/martian/v3 v3.2.1/go.mod h1:oBOf6HBosgwRXnUGWUB05QECsc6uvmMiJ3+6W4l/CUk= github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= @@ -273,45 +309,61 @@ github.com/google/pprof v0.0.0-20201023163331-3e6fc7fc9c4c/go.mod h1:kpwsk12EmLe github.com/google/pprof v0.0.0-20201203190320-1bf35d6f28c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20210122040257-d980be63207e/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20210226084205-cbba55b83ad5/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20210601050228-01bbb1931b22/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20210609004039-a478d1d731e9/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.5/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= +github.com/googleapis/gax-go/v2 v2.1.0/go.mod h1:Q3nei7sK6ybPYH7twZdmQpAd1MKb7pfu6SK+H1/DsU0= +github.com/googleapis/gax-go/v2 v2.1.1/go.mod h1:hddJymUZASv3XPyGkUpKj8pPO47Rmb0eJc8R6ouapiM= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= github.com/gorilla/websocket v1.4.1/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/graph-gophers/graphql-go v0.0.0-20201113091052-beb923fada29/go.mod h1:9CQHMSxwO4MprSdzoIEobiHpoLtHm77vfxsvsIN5Vuc= github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= -github.com/hashicorp/consul/api v1.1.0/go.mod h1:VmuI/Lkw1nC05EYQWNKwWGbkg+FbDBtguAZLlVdkD9Q= -github.com/hashicorp/consul/sdk v0.1.1/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8= +github.com/hashicorp/consul/api v1.11.0/go.mod h1:XjsvQN+RJGWI2TWy1/kqaE16HrR2J/FWgkYjdZQsX9M= +github.com/hashicorp/consul/sdk v0.8.0/go.mod h1:GBvyrGALthsZObzUGsfgHZQDXjg4lOjagTIwIR1vPms= github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= github.com/hashicorp/go-bexpr v0.1.10/go.mod h1:oxlubA2vC/gFVfX1A6JGp7ls7uCDlfJn732ehYYg+g0= +github.com/hashicorp/go-cleanhttp v0.5.0/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= +github.com/hashicorp/go-cleanhttp v0.5.2/go.mod h1:kO/YDlP8L1346E6Sodw+PrpBSV4/SoxCXGY6BqNFT48= +github.com/hashicorp/go-hclog v0.12.0/go.mod h1:whpDNt7SSdeAju8AWKIWsul05p54N/39EeqMAyrmvFQ= +github.com/hashicorp/go-hclog v1.0.0/go.mod h1:whpDNt7SSdeAju8AWKIWsul05p54N/39EeqMAyrmvFQ= github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= +github.com/hashicorp/go-immutable-radix v1.3.1/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM= github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk= -github.com/hashicorp/go-rootcerts v1.0.0/go.mod h1:K6zTfqpRlCUIjkwsN4Z+hiSfzSTQa6eBIzfwKfwNnHU= +github.com/hashicorp/go-multierror v1.1.0/go.mod h1:spPvp8C1qA32ftKqdAHm4hHTbPw+vmowP0z+KUhOZdA= +github.com/hashicorp/go-retryablehttp v0.5.3/go.mod h1:9B5zBasrRhHXnJnui7y6sL7es7NDiJgTc6Er0maI1Xs= +github.com/hashicorp/go-rootcerts v1.0.2/go.mod h1:pqUvnprVnM5bf7AOirdbb01K4ccR319Vf4pU3K5EGc8= github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU= github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4= github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= -github.com/hashicorp/go.net v0.0.1/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA0oac0k90= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= github.com/hashicorp/golang-lru v0.5.5-0.20210104140557-80c98217689d/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64= -github.com/hashicorp/mdns v1.0.0/go.mod h1:tL+uN++7HEJ6SQLQ2/p+z2pH24WQKWjBPkE0mNTz8vQ= -github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I= -github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc= +github.com/hashicorp/mdns v1.0.1/go.mod h1:4gW7WsVCke5TE7EPeYliwHlRUyBtfCwuFwuMg2DmyNY= +github.com/hashicorp/mdns v1.0.4/go.mod h1:mtBihi+LeNXGtG8L9dX59gAEa12BDtBQSp4v/YAJqrc= +github.com/hashicorp/memberlist v0.2.2/go.mod h1:MS2lj3INKhZjWNqd3N0m3J+Jxf3DAOnAH9VT3Sh9MUE= +github.com/hashicorp/memberlist v0.3.0/go.mod h1:MS2lj3INKhZjWNqd3N0m3J+Jxf3DAOnAH9VT3Sh9MUE= +github.com/hashicorp/serf v0.9.5/go.mod h1:UWDWwZeL5cuWDJdl0C6wrvrUwEqtQ4ZKBKKENpqIUyk= +github.com/hashicorp/serf v0.9.6/go.mod h1:TXZNMjZQijwlDvp+r0b63xZ45H7JmCmgg4gpTwn9UV4= github.com/holiman/bloomfilter/v2 v2.0.3/go.mod h1:zpoh+gs7qcpqrHr3dB55AMiJwo0iURXE7ZOP9L9hSkA= github.com/holiman/uint256 v1.2.0/go.mod h1:y4ga/t+u+Xwd7CpDgZESaRcWy0I7XMlTMA25ApIH5Jw= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/huin/goupnp v1.0.2/go.mod h1:0dxJBVBHqTMjIUMkESDTNgOOx/Mw5wYIfyFmdzSamkM= github.com/huin/goutil v0.0.0-20170803182201-1ca381bf3150/go.mod h1:PpLOETDnJ0o3iZrZfqZzyLl6l7F3c6L1oWn7OICBi6o= +github.com/iancoleman/strcase v0.2.0/go.mod h1:iwCmte+B7n89clKwxIoIXy/HfoL7AsD47ZCWhYzw7ho= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM= @@ -335,7 +387,9 @@ github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHW github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= github.com/jrick/logrotate v1.0.0/go.mod h1:LNinyqDIJnpAur+b8yyulnQw/wDuN1+BYKlTRt3OuAQ= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= +github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= github.com/jsternberg/zap-logfmt v1.0.0/go.mod h1:uvPs/4X51zdkcm5jXl5SYoN+4RK21K8mysFmDaM/h+o= @@ -358,6 +412,7 @@ github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxv github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.2.1 h1:Fmg33tUaq4/8ym9TJN1x7sLJnHVwhP33CNkpYV/7rwI= github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= @@ -371,6 +426,7 @@ github.com/leanovate/gopter v0.2.9/go.mod h1:U2L/78B+KVFIx2VmW6onHJQzXtFb+p5y3y2 github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/lucasjones/reggen v0.0.0-20180717132126-cdb49ff09d77 h1:6xiz3+ZczT3M4+I+JLpcPGG1bQKm8067HktB17EDWEE= github.com/lucasjones/reggen v0.0.0-20180717132126-cdb49ff09d77/go.mod h1:5ELEyG+X8f+meRWHuqUOewBOhvHkl7M76pdGEansxW4= +github.com/lyft/protoc-gen-star v0.5.3/go.mod h1:V0xaHgaf5oCCqmcxYcWiDfTiKsZsRc87/1qhoTACD8w= github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/magiconair/properties v1.8.5/go.mod h1:y3VJvCyxH9uVvJTWEGAELF3aiYNyPKd5NZ3oSwXrF60= github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= @@ -378,16 +434,21 @@ github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN github.com/matryer/moq v0.0.0-20190312154309-6cfb0558e1bd/go.mod h1:9ELz6aaclSIGnZBoaSLZ3NAl1VTufbOrXBPvtcy6WiQ= github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= +github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= +github.com/mattn/go-colorable v0.1.6/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= github.com/mattn/go-colorable v0.1.7/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= github.com/mattn/go-colorable v0.1.8/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= -github.com/mattn/go-colorable v0.1.9 h1:sqDoxXbdeALODt0DAeJCVp38ps9ZogZEAXjus69YV3U= github.com/mattn/go-colorable v0.1.9/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= +github.com/mattn/go-colorable v0.1.12 h1:jF+Du6AlPIjs2BiUiQlKOX0rt3SujHxPnksPKZbaA40= +github.com/mattn/go-colorable v0.1.12/go.mod h1:u5H1YNBxpqRaxsYJYSkiCWKzEfiAb1Gb520KVy5xxl4= github.com/mattn/go-ieproxy v0.0.0-20190610004146-91bb50d98149/go.mod h1:31jz6HNzdxOmlERGGEc4v/dMssOfmp2p5bT/okiKFFc= github.com/mattn/go-ieproxy v0.0.0-20190702010315-6dee0af9227d/go.mod h1:31jz6HNzdxOmlERGGEc4v/dMssOfmp2p5bT/okiKFFc= github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.9/go.mod h1:YNRxwqDuOph6SZLI9vUUz6OYw3QyUt7WiY2yME+cCiQ= +github.com/mattn/go-isatty v0.0.10/go.mod h1:qgIWMr58cqv1PHHyhnkY9lrL7etaEgOFcMEpPG5Rm84= +github.com/mattn/go-isatty v0.0.11/go.mod h1:PhnuNfih5lzO57/f3n+odYbM4JtupLOxQOAqxQCu2WE= github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= github.com/mattn/go-isatty v0.0.14 h1:yVuAays6BHfxijgZPzw+3Zlu5yQgKGP2/hcQbHb7S9Y= github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= @@ -398,12 +459,11 @@ github.com/mattn/go-sqlite3 v1.11.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsO github.com/mattn/go-tty v0.0.0-20180907095812-13ff1204f104/go.mod h1:XPvLUNfbS4fJH25nqRHfWLMa1ONC8Amw+mIA639KxkE= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= -github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc= -github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= +github.com/miekg/dns v1.1.26/go.mod h1:bPDLeHnStXmXAq1m/Ch/hvfNHr14JKNPMBo3VZKjuso= +github.com/miekg/dns v1.1.41/go.mod h1:p6aan82bvRIyn+zDIv9xYNUpwa73JcSh9BKwknJysuI= +github.com/mitchellh/cli v1.1.0/go.mod h1:xcISNoH86gajksDmfB23e/pu+B+GeFRMYmoHXxx3xhI= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI= -github.com/mitchellh/gox v0.4.0/go.mod h1:Sd9lOJ0+aimLBi73mGofS1ycjY8lL3uZM3JPS42BGNg= -github.com/mitchellh/iochan v1.0.0/go.mod h1:JwYml1nuB7xOzsp52dPpHFffvOCDupsG0QubkSMEySY= github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/mitchellh/mapstructure v1.4.1/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= @@ -414,6 +474,7 @@ github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJ github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= github.com/mschoch/smat v0.0.0-20160514031455-90eadee771ae/go.mod h1:qAyveg+e4CE+eKJXWVjKXM4ck2QobLqTDytGJbLLhJg= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/naoina/go-stringutil v0.1.0/go.mod h1:XJ2SJL9jCtBh+P9q5btrd/Ylo8XwT/h1USek5+NqSA0= @@ -436,9 +497,10 @@ github.com/opentracing/opentracing-go v1.0.2/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFSt github.com/opentracing/opentracing-go v1.0.3-0.20180606204148-bd9c31933947/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= +github.com/pascaldekloe/goe v0.1.0/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= github.com/paulbellamy/ratecounter v0.2.0/go.mod h1:Hfx1hDpSGoqxkVVpBi/IlYD7kChlfo5C6hzIHwPqfFE= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= -github.com/pelletier/go-toml v1.9.3/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= +github.com/pelletier/go-toml v1.9.4/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= github.com/peterh/liner v1.0.1-0.20180619022028-8c1271fcf47f/go.mod h1:xIteQHvHuaLYG9IFj6mSxM0fCKrs34IrEQUhOYuGPHc= github.com/peterh/liner v1.1.1-0.20190123174540-a2c9a5303de7/go.mod h1:CRroGNssyjTd/qIG2FyxByd2S8JEAZXBl4qUrZf8GS0= github.com/philhofer/fwd v1.0.0/go.mod h1:gk3iGcWd9+svBvR0sR+KPcfE+RNWozjowpeBVG3ZVNU= @@ -452,16 +514,21 @@ github.com/pkg/term v0.0.0-20180730021639-bffc007b7fd5/go.mod h1:eCbImbZ95eXtAUI github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI= +github.com/posener/complete v1.2.3/go.mod h1:WZIdtGGp+qx0sLrYKtIRAruyNpv6hFCicSgv7Sy7s/s= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= +github.com/prometheus/client_golang v1.4.0/go.mod h1:e9GMxYsXl05ICDXkRhurwBS4Q3OK1iX/F2sw+iXX5zU= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.6.0/go.mod h1:eBmuwkDJBwy6iBfxCBob6t6dR6ENT/y+J+Zk0j9GMYc= +github.com/prometheus/common v0.9.1/go.mod h1:yhUN8i9wzaXS3w1O07YhxHEBxD+W35wd8bs7vj7HSQ4= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= +github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A= github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= github.com/retailnext/hllpp v1.0.1-0.20180308014038-101a6d2f8b52/go.mod h1:RDpi1RftBQPUCDRw6SmxeaREsAaRKnOclghuzp/WRzc= github.com/rjeczalik/notify v0.9.1/go.mod h1:rKwnCoCGeuQnwBtTSPL9Dad03Vh2n40ePRrjvIXnJho= @@ -470,7 +537,9 @@ github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFR github.com/rs/cors v1.7.0/go.mod h1:gFx+x8UowdsKA9AchylcLynDq+nNFfI8FkUZdN/jGCU= github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= +github.com/sagikazarmark/crypt v0.3.0/go.mod h1:uD/D+6UF4SrIR1uGEv7bBNkNqLGqUr43MRiaGWX1Nig= github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= github.com/segmentio/fasthash v1.0.3 h1:EI9+KE1EwvMLBWwjpRDc+fEM+prwxDYbslddQGtrmhM= github.com/segmentio/fasthash v1.0.3/go.mod h1:waKX8l2N8yckOgmSsXJi7x1ZfdKZ4x7KRMzBtS3oedY= @@ -480,26 +549,28 @@ github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAm github.com/shirou/gopsutil v3.21.4-0.20210419000835-c7a38de76ee5+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA= github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= +github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI= github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= +github.com/spf13/afero v1.3.3/go.mod h1:5KUK8ByomD5Ti5Artl0RtHeI5pTF7MIDuXL3yY520V4= github.com/spf13/afero v1.6.0/go.mod h1:Ai8FlHk4v/PARR026UzYexafAt9roJ7LcLMAmO6Z93I= github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= -github.com/spf13/cast v1.3.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= +github.com/spf13/cast v1.4.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU= -github.com/spf13/cobra v1.2.1 h1:+KmjbUw1hriSNMF55oPrkZcb27aECyrj8V2ytv7kWDw= -github.com/spf13/cobra v1.2.1/go.mod h1:ExllRjgxM/piMAM+3tAZvg8fsklGAf3tPfi+i8t68Nk= +github.com/spf13/cobra v1.3.0 h1:R7cSvGu+Vv+qX0gW5R/85dx2kmmJT5z5NM8ifdYjdn0= +github.com/spf13/cobra v1.3.0/go.mod h1:BrRVncBjOJa/eUcVVm9CE+oC6as8k+VYr4NY7WCi9V4= github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= github.com/spf13/jwalterweatherman v1.1.0/go.mod h1:aNWZUN0dPAAO/Ljvb5BEdw96iTZ0EXowPYD95IqWIGo= github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s= -github.com/spf13/viper v1.8.1/go.mod h1:o0Pch8wJ9BVSWGQMbra6iw0oQ5oktSIBaujf1rJH9Ns= +github.com/spf13/viper v1.10.0/go.mod h1:SoyBPwAtKDzypXNDFKN5kzH7ppppbGZtls1UpIy5AsM= github.com/status-im/keycard-go v0.0.0-20190316090335-8537d3370df4/go.mod h1:RZLeN1LMWmRsyYjvAu+I6Dm9QmlDaIIt+Y+4Kd7Tp+Q= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1 h1:2vfRuCMp5sSVIDSqO8oNnWJq7mPa6KVP3iPIwFBuy8A= @@ -526,6 +597,7 @@ github.com/tidwall/sjson v1.2.3/go.mod h1:5WdjKx3AQMvCJ4RG6/2UYT7dLrGvJUV1x4jdTA github.com/tinylib/msgp v1.0.2/go.mod h1:+d+yLhGm8mzTaHzB+wgMYrodPfmZrzkirds8fDWklFE= github.com/tklauser/go-sysconf v0.3.5/go.mod h1:MkWzOF4RMCshBAMXuhXJs64Rte09mITnppBXY/rYEFI= github.com/tklauser/numcpus v0.2.2/go.mod h1:x3qojaO3uyYt0i56EW/VUYs7uBvdl2fkfZFu0T9wgjM= +github.com/tv42/httpunix v0.0.0-20150427012821-b75d8614f926/go.mod h1:9ESjWnEqriFuLhtthL60Sar/7RFoluCcXsuvEwTV5KM= github.com/tyler-smith/go-bip39 v1.0.1-0.20181017060643-dbb3b84ba2ef/go.mod h1:sJ5fKU0s6JVwZjjcUEX2zFOnvq0ASQ2K9Zr6cf67kNs= github.com/tyler-smith/go-bip39 v1.0.2/go.mod h1:sJ5fKU0s6JVwZjjcUEX2zFOnvq0ASQ2K9Zr6cf67kNs= github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= @@ -546,9 +618,9 @@ github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= -go.etcd.io/etcd/api/v3 v3.5.0/go.mod h1:cbVKeC6lCfl7j/8jBhAK6aIYO9XOjdptoxU/nLQcPvs= -go.etcd.io/etcd/client/pkg/v3 v3.5.0/go.mod h1:IJHfcCEKxYu1Os13ZdwCwIUTUVGYTSAM3YSwc9/Ac1g= -go.etcd.io/etcd/client/v2 v2.305.0/go.mod h1:h9puh54ZTgAKtEbut2oe9P4L/oqKCVB6xsXlzd7alYQ= +go.etcd.io/etcd/api/v3 v3.5.1/go.mod h1:cbVKeC6lCfl7j/8jBhAK6aIYO9XOjdptoxU/nLQcPvs= +go.etcd.io/etcd/client/pkg/v3 v3.5.1/go.mod h1:IJHfcCEKxYu1Os13ZdwCwIUTUVGYTSAM3YSwc9/Ac1g= +go.etcd.io/etcd/client/v2 v2.305.1/go.mod h1:pMEacxZW7o8pg4CrFE7pquyCJJzZvkvdD2RibOCCCGs= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= @@ -556,6 +628,7 @@ go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E= +go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= @@ -572,14 +645,16 @@ golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190909091759-094676da4a83/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190923035154-9ee001bba392/go.mod h1:/lpIB1dKB+9EgE3H3cr1v9wB50oz8l4C4h62xy7jSTY= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200115085410-6d4e4cb37c7d/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200510223506-06a226fb4e37/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200820211705-5c72a883971a/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20201221181555-eec23a3978ad/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= -golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2 h1:It14KIkyBFYkHkwZ7k45minvA9aorojkyjGk9KJ5B/w= golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= +golang.org/x/crypto v0.0.0-20210817164053-32db794688a5 h1:HWj/xjIHfjYU5nVXpTM0s39J9CbLn7Cc5a7IC5rwsMQ= +golang.org/x/crypto v0.0.0-20210817164053-32db794688a5/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20180807140117-3d87b88a115f/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= @@ -619,13 +694,13 @@ golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.5.0/go.mod h1:5OXOZSfqPIIbmVBIIKWRFfZjPR0E5r58TLhUjH0a2Ro= golang.org/x/net v0.0.0-20180719180050-a680a1efc54d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181023162649-9b4f9f5ad519/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= @@ -637,6 +712,7 @@ golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190923162816-aa69164e4478/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= @@ -662,8 +738,11 @@ golang.org/x/net v0.0.0-20210220033124-5f55cee0dc0d/go.mod h1:m0MpNAwzfU5UDzcl9v golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210316092652-d523dce5a7f4/go.mod h1:RBQZq4jEuRlivfhVLdyRGr576XBO4/greRjx4P4O3yc= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= -golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d h1:20cMwl2fHAzkJMEA+8J4JgqBQcQGzbisXo31MIeenXI= +golang.org/x/net v0.0.0-20210410081132-afb366fc7cd1/go.mod h1:9tjilg8BloeKEkVJvy7fQ90B1CfIiPueXVOjqfkSzI8= +golang.org/x/net v0.0.0-20210503060351-7fd8e65b6420/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20210813160813-60bc85c4be6d h1:LO7XpTYMwTqxjLcGWPijK3vRXg1aWdlNOVOHRq45d7c= +golang.org/x/net v0.0.0-20210813160813-60bc85c4be6d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -675,7 +754,12 @@ golang.org/x/oauth2 v0.0.0-20201208152858-08078c50e5b5/go.mod h1:KelEdhl1UZF7XfJ golang.org/x/oauth2 v0.0.0-20210218202405-ba52d332ba99/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20210220000619-9bb904979d93/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20210313182246-cd4f82c27b84/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20210402161424-2e8d93401602/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210514164344-f6687ab2804c/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210628180205-a41e5a781914/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210805134026-6f1e6394065a/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210819190943-2bc19b11175f/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20211005180243-6b3c2da341f1/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -700,6 +784,7 @@ golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5h golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -708,8 +793,11 @@ golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190922100055-0a153f010e69/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190924154521-2837fb4f24fe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191008105621-543471e840be/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -718,6 +806,7 @@ golang.org/x/sys v0.0.0-20200107162124-548cf772de50/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200124204421-9fbb57f87de9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -740,6 +829,7 @@ golang.org/x/sys v0.0.0-20210104204734-6f8348627aad/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210220050731-9a76102bfb43/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210303074136-134d130e1a04/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210305230114-8fe3ee5dd75b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210315160823-c6e025ad8005/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210316164454-77fc1eacc6aa/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -750,9 +840,20 @@ golang.org/x/sys v0.0.0-20210403161142-5e06dd20ab57/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210420205809-ac73e9fd8988/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210514084401-e8d321eab015/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210603125802-9665404d3644/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210816183151-1e6c022a8912 h1:uCLL3g5wH2xjxVREVuAbP9JM5PPKjRbXKRa6IBjkzmU= +golang.org/x/sys v0.0.0-20210806184541-e5e7981a1069/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210816183151-1e6c022a8912/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210823070655-63515b42dcdf/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210908233432-aa78b53d3365/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211124211545-fe61309f8881/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211205182925-97ca703d548d h1:FjkYO/PPp4Wi0EAUOVLxePm7qVW4r4ctbWpURyuOD0E= +golang.org/x/sys v0.0.0-20211205182925-97ca703d548d/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -763,6 +864,7 @@ golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -785,9 +887,9 @@ golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgw golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20190907020128-2ca718005c18/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191112195655-aa38f8e97acc/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= @@ -822,7 +924,11 @@ golang.org/x/tools v0.0.0-20201208233053-a543418bbed2/go.mod h1:emZCQorbCU4vsT4f golang.org/x/tools v0.0.0-20210105154028-b0ab187a4818/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= +golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/tools v0.1.3/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/tools v0.1.4/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -855,7 +961,17 @@ google.golang.org/api v0.36.0/go.mod h1:+z5ficQTmoYpPn8LCUNVpK5I7hwkpjbcgqA7I34q google.golang.org/api v0.40.0/go.mod h1:fYKFpnQN0DsDSKRVRcQSDQNtqWPfM9i+zNPxepjRCQ8= google.golang.org/api v0.41.0/go.mod h1:RkxM5lITDfTzmyKFPt+wGrCJbVfniCr2ool8kTBzRTU= google.golang.org/api v0.43.0/go.mod h1:nQsDGjRXMo4lvh5hP0TKqF244gqhGcr/YSIykhUk/94= -google.golang.org/api v0.44.0/go.mod h1:EBOGZqzyhtvMDoxwS97ctnh0zUmYY6CxqXsc1AvkYD8= +google.golang.org/api v0.47.0/go.mod h1:Wbvgpq1HddcWVtzsVLyfLp8lDg6AA241LmgIL59tHXo= +google.golang.org/api v0.48.0/go.mod h1:71Pr1vy+TAZRPkPs/xlCf5SsU8WjuAWv1Pfjbtukyy4= +google.golang.org/api v0.50.0/go.mod h1:4bNT5pAuq5ji4SRZm+5QIkjny9JAyVD/3gaSihNefaw= +google.golang.org/api v0.51.0/go.mod h1:t4HdrdoNgyN5cbEfm7Lum0lcLDLiise1F8qDKX00sOU= +google.golang.org/api v0.54.0/go.mod h1:7C4bFFOvVDGXjfDTAsgGwDgAxRDeQ4X8NvUedIt6z3k= +google.golang.org/api v0.55.0/go.mod h1:38yMfeP1kfjsl8isn0tliTjIb1rJXcQi4UXlbqivdVE= +google.golang.org/api v0.56.0/go.mod h1:38yMfeP1kfjsl8isn0tliTjIb1rJXcQi4UXlbqivdVE= +google.golang.org/api v0.57.0/go.mod h1:dVPlbZyBo2/OjBpmvNdpn2GRm6rPy75jyU7bmhdrMgI= +google.golang.org/api v0.59.0/go.mod h1:sT2boj7M9YJxZzgeZqXogmhfmRWDtPzT31xkieUbuZU= +google.golang.org/api v0.61.0/go.mod h1:xQRti5UdCmoCEqFxcz93fTl338AVqDgyaDRuOZ3hg9I= +google.golang.org/api v0.62.0/go.mod h1:dKmwPCydfsad4qCH08MSdgWjfHOyfpd4VtDGgRFdavw= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= @@ -905,7 +1021,29 @@ google.golang.org/genproto v0.0.0-20210303154014-9728d6b83eeb/go.mod h1:FWY/as6D google.golang.org/genproto v0.0.0-20210310155132-4ce2db91004e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20210319143718-93e7006c17a6/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20210402141018-6c239bbf2bb1/go.mod h1:9lPAdzaEmUacj36I+k7YKbEc5CXzPIeORRgDAUOu28A= +google.golang.org/genproto v0.0.0-20210513213006-bf773b8c8384/go.mod h1:P3QM42oQyzQSnHPnZ/vqoCdDmzH28fzWByN9asMeM8A= google.golang.org/genproto v0.0.0-20210602131652-f16073e35f0c/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0= +google.golang.org/genproto v0.0.0-20210604141403-392c879c8b08/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0= +google.golang.org/genproto v0.0.0-20210608205507-b6d2f5bf0d7d/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0= +google.golang.org/genproto v0.0.0-20210624195500-8bfb893ecb84/go.mod h1:SzzZ/N+nwJDaO1kznhnlzqS8ocJICar6hYhVyhi++24= +google.golang.org/genproto v0.0.0-20210713002101-d411969a0d9a/go.mod h1:AxrInvYm1dci+enl5hChSFPOmmUF1+uAa/UsgNRWd7k= +google.golang.org/genproto v0.0.0-20210716133855-ce7ef5c701ea/go.mod h1:AxrInvYm1dci+enl5hChSFPOmmUF1+uAa/UsgNRWd7k= +google.golang.org/genproto v0.0.0-20210728212813-7823e685a01f/go.mod h1:ob2IJxKrgPT52GcgX759i1sleT07tiKowYBGbczaW48= +google.golang.org/genproto v0.0.0-20210805201207-89edb61ffb67/go.mod h1:ob2IJxKrgPT52GcgX759i1sleT07tiKowYBGbczaW48= +google.golang.org/genproto v0.0.0-20210813162853-db860fec028c/go.mod h1:cFeNkxwySK631ADgubI+/XFU/xp8FD5KIVV4rj8UC5w= +google.golang.org/genproto v0.0.0-20210821163610-241b8fcbd6c8/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= +google.golang.org/genproto v0.0.0-20210828152312-66f60bf46e71/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= +google.golang.org/genproto v0.0.0-20210831024726-fe130286e0e2/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= +google.golang.org/genproto v0.0.0-20210903162649-d08c68adba83/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= +google.golang.org/genproto v0.0.0-20210909211513-a8c4777a87af/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= +google.golang.org/genproto v0.0.0-20210924002016-3dee208752a0/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= +google.golang.org/genproto v0.0.0-20211008145708-270636b82663/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= +google.golang.org/genproto v0.0.0-20211028162531-8db9c33dc351/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= +google.golang.org/genproto v0.0.0-20211118181313-81c1377c94b1/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= +google.golang.org/genproto v0.0.0-20211129164237-f09f9a12af12/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= +google.golang.org/genproto v0.0.0-20211203200212-54befc351ae9/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= +google.golang.org/genproto v0.0.0-20211206160659-862468c7d6e0/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= +google.golang.org/genproto v0.0.0-20211208223120-3a66f561d7aa/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= @@ -925,7 +1063,15 @@ google.golang.org/grpc v1.34.0/go.mod h1:WotjhfgOW/POjDeRt8vscBtXq+2VjORFy659qA5 google.golang.org/grpc v1.35.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= google.golang.org/grpc v1.36.1/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= +google.golang.org/grpc v1.37.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= +google.golang.org/grpc v1.37.1/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= google.golang.org/grpc v1.38.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= +google.golang.org/grpc v1.39.0/go.mod h1:PImNr+rS9TWYb2O4/emRugxiyHZ5JyHW5F+RPnDzfrE= +google.golang.org/grpc v1.39.1/go.mod h1:PImNr+rS9TWYb2O4/emRugxiyHZ5JyHW5F+RPnDzfrE= +google.golang.org/grpc v1.40.0/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34= +google.golang.org/grpc v1.40.1/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34= +google.golang.org/grpc v1.42.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU= +google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.1.0/go.mod h1:6Kw0yEErY5E/yWrBtf03jp27GLLJujG4z/JK95pnjjw= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= @@ -937,8 +1083,9 @@ google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpAD google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4= google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= -google.golang.org/protobuf v1.26.0 h1:bxAC2xTBsZGibn2RTntX0oH50xLsqy1OxA9tTL3p/lk= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +google.golang.org/protobuf v1.27.1 h1:SnqbnDw1V7RiZcXPx5MEeqPv2s79L9i7BJUlG/+RurQ= +google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= @@ -947,7 +1094,7 @@ gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntN gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= -gopkg.in/ini.v1 v1.62.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= +gopkg.in/ini.v1 v1.66.2/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/natefinch/npipe.v2 v2.0.0-20160621034901-c1b8fa8bdcce/go.mod h1:5AcXVHNjg+BDxry382+8OKon8SEWiKktQR07RKPsv1c= gopkg.in/olebedev/go-duktape.v3 v3.0.0-20200619000410-60c24ae608a6/go.mod h1:uAJfkITjFhyEEuUfm7bsmCZRbW5WRq8s9EY8HZ6hCns= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= @@ -956,6 +1103,7 @@ gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= From 380b82635e64d78d5ae6c7126bd4954b4357d5ed Mon Sep 17 00:00:00 2001 From: Tyler Hale <7747991+t-hale@users.noreply.github.com> Date: Tue, 25 Jan 2022 16:55:45 -0500 Subject: [PATCH 288/376] fixes #269 (#270) --- pkg/results/construction_results.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pkg/results/construction_results.go b/pkg/results/construction_results.go index 32915338..4761a538 100644 --- a/pkg/results/construction_results.go +++ b/pkg/results/construction_results.go @@ -313,7 +313,9 @@ func ExitConstruction( ) if results != nil { results.Print() - results.Output(config.Construction.ResultsOutputFile) + if config.Construction != nil { + results.Output(config.Construction.ResultsOutputFile) + } } return err From cf9632d4d553fe892b12d2fd0b5d5d531a4085ee Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 11 Feb 2022 09:15:17 -0800 Subject: [PATCH 289/376] Bump github.com/coinbase/rosetta-sdk-go from 0.7.2 to 0.7.3 (#273) Bumps [github.com/coinbase/rosetta-sdk-go](https://github.com/coinbase/rosetta-sdk-go) from 0.7.2 to 0.7.3. - [Release notes](https://github.com/coinbase/rosetta-sdk-go/releases) - [Commits](https://github.com/coinbase/rosetta-sdk-go/compare/v0.7.2...v0.7.3) --- updated-dependencies: - dependency-name: github.com/coinbase/rosetta-sdk-go dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 29 ++++++++++++++++++++++++++--- 2 files changed, 27 insertions(+), 4 deletions(-) diff --git a/go.mod b/go.mod index 6308774e..ba515ab0 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/coinbase/rosetta-cli go 1.16 require ( - github.com/coinbase/rosetta-sdk-go v0.7.2 + github.com/coinbase/rosetta-sdk-go v0.7.3 github.com/fatih/color v1.13.0 github.com/olekukonko/tablewriter v0.0.5 github.com/pkg/errors v0.9.1 diff --git a/go.sum b/go.sum index 718bbfe4..3b1265fa 100644 --- a/go.sum +++ b/go.sum @@ -50,6 +50,10 @@ cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RX cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= collectd.org v0.3.0/go.mod h1:A/8DzQBkF6abtvrT2j/AU/4tiBgJWYyh0y/oB/4MlWE= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= +filippo.io/edwards25519 v1.0.0-rc.1 h1:m0VOOB23frXZvAOK44usCgLWvtsxIoMCTBGJZlpmGfU= +filippo.io/edwards25519 v1.0.0-rc.1/go.mod h1:N1IkdkCkiLB6tki+MYJoSx2JTY9NUlxZE7eHn5EwJns= +git.sr.ht/~sircmpwn/getopt v0.0.0-20191230200459-23622cc906b3/go.mod h1:wMEGFFFNuPos7vHmWXfszqImLppbc0wEhh6JBfJIUgw= +git.sr.ht/~sircmpwn/go-bare v0.0.0-20210406120253-ab86bc2846d9/go.mod h1:BVJwbDfVjCjoFiKrhkei6NdGcZYpkDkdyCdg1ukytRA= github.com/Azure/azure-pipeline-go v0.2.1/go.mod h1:UGSo8XybXnIGZ3epmeBw7Jdz+HiUVpqIlpz/HKHylF4= github.com/Azure/azure-pipeline-go v0.2.2/go.mod h1:4rQ/NZncSvGqNkkOsNpOU1tgoNuIlp9AfUH5G1tvCHc= github.com/Azure/azure-storage-blob-go v0.7.0/go.mod h1:f9YQKtsG1nMisotuTPpO0tjNuEjKRYAcJU8/ydDI++4= @@ -108,11 +112,14 @@ github.com/bmizerany/pat v0.0.0-20170815010413-6226ea591a40/go.mod h1:8rLXio+Wji github.com/boltdb/bolt v1.3.1/go.mod h1:clJnj/oiGkjum5o1McbSZDSLxVThjynRyGBgiAx27Ps= github.com/btcsuite/btcd v0.0.0-20190315201642-aa6e0f35703c/go.mod h1:DrZx5ec/dmnfpw9KyYoQyYo7d0KEvTkk/5M/vbZjAr8= github.com/btcsuite/btcd v0.20.1-beta/go.mod h1:wVuoA8VJLEcwgqHBwHmzLRazpKxTv13Px/pDuV7OomQ= +github.com/btcsuite/btcd v0.21.0-beta.0.20201114000516-e9c7a5ac6401/go.mod h1:Sv4JPQ3/M+teHz9Bo5jBpkNcP0x6r7rdihlNL/7tTAs= github.com/btcsuite/btcd v0.22.0-beta h1:LTDpDKUM5EeOFBPM8IXpinEcmZ6FWfNZbE3lfrfdnWo= github.com/btcsuite/btcd v0.22.0-beta/go.mod h1:9n5ntfhhHQBIhUvlhDvD3Qg6fRUj4jkN0VB8L8svzOA= github.com/btcsuite/btclog v0.0.0-20170628155309-84c8d2346e9f/go.mod h1:TdznJufoqS23FtqVCzL0ZqgP5MqXbb4fg/WgDys70nA= github.com/btcsuite/btcutil v0.0.0-20190207003914-4c204d697803/go.mod h1:+5NJ2+qvTyV9exUAL/rxXi3DcLg2Ts+ymUAY5y4NvMg= github.com/btcsuite/btcutil v0.0.0-20190425235716-9e5f4b9a998d/go.mod h1:+5NJ2+qvTyV9exUAL/rxXi3DcLg2Ts+ymUAY5y4NvMg= +github.com/btcsuite/btcutil v1.0.2/go.mod h1:j9HUFwoQRsZL3V4n+qG+CUnEGHOarIxfC3Le2Yhbcts= +github.com/btcsuite/btcutil v1.0.3-0.20201208143702-a53e38424cce h1:YtWJF7RHm2pYCvA5t0RPmAaLUhREsKuKd+SLhxFbFeQ= github.com/btcsuite/btcutil v1.0.3-0.20201208143702-a53e38424cce/go.mod h1:0DVlHczLPewLcPGEIeUEzfOJhqGPQ0mJJRDBtD307+o= github.com/btcsuite/go-socks v0.0.0-20170105172521-4720035b7bfd/go.mod h1:HHNXQzUsZCxOoE+CPiyCTO6x34Zs86zZUiwtpXoGdtg= github.com/btcsuite/goleveldb v0.0.0-20160330041536-7834afc9e8cd/go.mod h1:F+uVaaLLH7j4eDXPRvw78tMflu7Ie2bzYOH4Y8rRKBY= @@ -121,6 +128,8 @@ github.com/btcsuite/snappy-go v0.0.0-20151229074030-0bdef8d06723/go.mod h1:8woku github.com/btcsuite/snappy-go v1.0.0/go.mod h1:8woku9dyThutzjeg+3xrA5iCpBRH8XEEg3lh6TiUghc= github.com/btcsuite/websocket v0.0.0-20150119174127-31079b680792/go.mod h1:ghJtEyQwv5/p4Mg4C0fgbePVuGr935/5ddU9Z3TmDRY= github.com/btcsuite/winsvc v1.0.0/go.mod h1:jsenWakMcC0zFBFurPLEAyrnc/teJEM1O46fmI40EZs= +github.com/bwesterb/go-ristretto v1.2.0 h1:xxWOVbN5m8NNKiSDZXE1jtZvZnC6JSJ9cYFADiZcWtw= +github.com/bwesterb/go-ristretto v1.2.0/go.mod h1:fUIoIZaG73pV5biE2Blr2xEzDoMj7NFEuV9ekS419A0= github.com/c-bata/go-prompt v0.2.2/go.mod h1:VzqtzE2ksDBcdln8G7mk2RX9QyGjH+OVqOCSiVIqS34= github.com/cenkalti/backoff v2.2.1+incompatible h1:tNowT99t7UNflLxfYYSlKYsBpXdEet03Pg2g16Swow4= github.com/cenkalti/backoff v2.2.1+incompatible/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM= @@ -148,10 +157,15 @@ github.com/cncf/xds/go v0.0.0-20210922020428-25de7278fc84/go.mod h1:eXthEFrGJvWH github.com/cncf/xds/go v0.0.0-20211001041855-01bcc9b48dfe/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cncf/xds/go v0.0.0-20211130200136-a8f946100490/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= -github.com/coinbase/rosetta-sdk-go v0.7.2 h1:uCNrASIyt7rV9bA3gzPG3JDlxVP5v/zLgi01GWngncM= -github.com/coinbase/rosetta-sdk-go v0.7.2/go.mod h1:wk9dvjZFSZiWSNkFuj3dMleTA1adLFotg5y71PhqKB4= +github.com/coinbase/kryptology v1.5.5 h1:iSB2AOGYBxvq+/4huDWVEnyCWWWrVBbb7yiCcLrE2rw= +github.com/coinbase/kryptology v1.5.5/go.mod h1:3zId6V3k/8v/o5gksjffbWMkH7Ik1UgeZVGTBvlpIxc= +github.com/coinbase/rosetta-sdk-go v0.7.3 h1:SnnJ5VJGZe6l4rYhyNha94SVUJinVwtZTPvtfgF2Xfw= +github.com/coinbase/rosetta-sdk-go v0.7.3/go.mod h1:PeO5OtybTPoddCrBrWaJS2vBICc2oLFoJkIIuRS/wPo= github.com/consensys/bavard v0.1.8-0.20210406032232-f3452dc9b572/go.mod h1:Bpd0/3mZuaj6Sj+PqrmIquiOKy397AKGThQPaGzNXAQ= +github.com/consensys/bavard v0.1.8-0.20210915155054-088da2f7f54a/go.mod h1:9ItSMtA/dXMAiL7BG6bqW2m3NdSEObYWoH223nGHukI= github.com/consensys/gnark-crypto v0.4.1-0.20210426202927-39ac3d4b3f1f/go.mod h1:815PAHg3wvysy0SyIqanF8gZ0Y1wjk/hrDHD/iT88+Q= +github.com/consensys/gnark-crypto v0.5.3 h1:4xLFGZR3NWEH2zy+YzvzHicpToQR8FXFbfLNvpGB+rE= +github.com/consensys/gnark-crypto v0.5.3/go.mod h1:hOdPlWQV1gDLp7faZVeg8Y0iEPFaOUnCc4XeCCk96p0= github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk= github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= @@ -326,6 +340,7 @@ github.com/gorilla/websocket v1.4.1/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/ad github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/graph-gophers/graphql-go v0.0.0-20201113091052-beb923fada29/go.mod h1:9CQHMSxwO4MprSdzoIEobiHpoLtHm77vfxsvsIN5Vuc= github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= +github.com/gtank/merlin v0.1.1/go.mod h1:T86dnYJhcGOh5BjZFCJWTDeTK7XW8uE+E21Cy/bIQ+s= github.com/hashicorp/consul/api v1.11.0/go.mod h1:XjsvQN+RJGWI2TWy1/kqaE16HrR2J/FWgkYjdZQsX9M= github.com/hashicorp/consul/sdk v0.8.0/go.mod h1:GBvyrGALthsZObzUGsfgHZQDXjg4lOjagTIwIR1vPms= github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= @@ -422,6 +437,7 @@ github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= github.com/labstack/echo/v4 v4.2.1/go.mod h1:AA49e0DZ8kk5jTOOCKNuPR6oTnBS0dYiM4FW1e6jwpg= github.com/labstack/gommon v0.3.0/go.mod h1:MULnywXg0yavhxWKc+lOruYdAhDwPK9wf0OL7NoOu+k= +github.com/leanovate/gopter v0.2.9 h1:fQjYxZaynp97ozCzfOyOuAGOU4aU/z37zf/tOujFk7c= github.com/leanovate/gopter v0.2.9/go.mod h1:U2L/78B+KVFIx2VmW6onHJQzXtFb+p5y3y2Sh+Jxxv8= github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/lucasjones/reggen v0.0.0-20180717132126-cdb49ff09d77 h1:6xiz3+ZczT3M4+I+JLpcPGG1bQKm8067HktB17EDWEE= @@ -461,6 +477,7 @@ github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5 github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= github.com/miekg/dns v1.1.26/go.mod h1:bPDLeHnStXmXAq1m/Ch/hvfNHr14JKNPMBo3VZKjuso= github.com/miekg/dns v1.1.41/go.mod h1:p6aan82bvRIyn+zDIv9xYNUpwa73JcSh9BKwknJysuI= +github.com/mimoo/StrobeGo v0.0.0-20181016162300-f8f6d4d2b643/go.mod h1:43+3pMjjKimDBf5Kr4ZFNGbLql1zKkbImw+fZbw3geM= github.com/mitchellh/cli v1.1.0/go.mod h1:xcISNoH86gajksDmfB23e/pu+B+GeFRMYmoHXxx3xhI= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI= @@ -481,6 +498,8 @@ github.com/naoina/go-stringutil v0.1.0/go.mod h1:XJ2SJL9jCtBh+P9q5btrd/Ylo8XwT/h github.com/naoina/toml v0.1.2-0.20170918210437-9fafd6967416/go.mod h1:NBIhNtsFMo3G2szEBne+bO4gS192HuIYRqfvOWb4i1E= github.com/neilotoole/errgroup v0.1.6 h1:PODGqPXdT5BC/zCYIMoTrwV+ujKcW+gBXM6Ye9Ve3R8= github.com/neilotoole/errgroup v0.1.6/go.mod h1:Q2nLGf+594h0CLBs/Mbg6qOr7GtqDK7C2S41udRnToE= +github.com/nixberg/chacha-rng-go v0.1.0 h1:2Y90hS8H/O+82yLWKD4MldkJM7Pzhezc7rT6EILvDNo= +github.com/nixberg/chacha-rng-go v0.1.0/go.mod h1:iPf1i6Vcwgoue86dblORobEXNCd1PFQiysFiImawfCM= github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= github.com/olekukonko/tablewriter v0.0.5 h1:P2Ga83D34wi1o9J6Wh1mRuqd4mF/x/lgBS7N7AbDhec= @@ -653,6 +672,7 @@ golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPh golang.org/x/crypto v0.0.0-20200820211705-5c72a883971a/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20201221181555-eec23a3978ad/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= +golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20210817164053-32db794688a5 h1:HWj/xjIHfjYU5nVXpTM0s39J9CbLn7Cc5a7IC5rwsMQ= golang.org/x/crypto v0.0.0-20210817164053-32db794688a5/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= @@ -741,8 +761,9 @@ golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96b golang.org/x/net v0.0.0-20210410081132-afb366fc7cd1/go.mod h1:9tjilg8BloeKEkVJvy7fQ90B1CfIiPueXVOjqfkSzI8= golang.org/x/net v0.0.0-20210503060351-7fd8e65b6420/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20210813160813-60bc85c4be6d h1:LO7XpTYMwTqxjLcGWPijK3vRXg1aWdlNOVOHRq45d7c= golang.org/x/net v0.0.0-20210813160813-60bc85c4be6d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f h1:OfiFi4JbukWwe3lzw+xunroH1mnC1e2Gy5cxNJApiSY= +golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -851,6 +872,7 @@ golang.org/x/sys v0.0.0-20210823070655-63515b42dcdf/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20210908233432-aa78b53d3365/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211019181941-9d821ace8654/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211124211545-fe61309f8881/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211205182925-97ca703d548d h1:FjkYO/PPp4Wi0EAUOVLxePm7qVW4r4ctbWpURyuOD0E= golang.org/x/sys v0.0.0-20211205182925-97ca703d548d/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -1123,3 +1145,4 @@ rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8 rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4= rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= +rsc.io/tmplfunc v0.0.3/go.mod h1:AG3sTPzElb1Io3Yg4voV9AGZJuleGAwaVRxL9M49PhA= From 48e9c9fc301ad54c32c8390ff6f6d54456ecd952 Mon Sep 17 00:00:00 2001 From: Madhur Shrimal Date: Wed, 16 Feb 2022 15:48:37 -0800 Subject: [PATCH 290/376] update circleci (#275) --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 12464874..232e169e 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -16,7 +16,7 @@ version: 2.1 executors: default: docker: - - image: circleci/golang:1.16 + - image: cimg/go:1.16 user: root # go directory is owned by root working_directory: /go/src/github.com/coinbase/rosetta-cli environment: From d7d68deb09f1b8a1150788510a0934356c81c0a7 Mon Sep 17 00:00:00 2001 From: Madhur Shrimal Date: Wed, 23 Feb 2022 17:36:20 -0700 Subject: [PATCH 291/376] add support for zap logger for structured logs (#276) * add zap logger * depds * add comments * remove unwanted lines --- go.mod | 1 + go.sum | 6 +++ pkg/logger/logger.go | 75 +++++++++++++++++++++++++++++++++++++- pkg/tester/construction.go | 7 +++- pkg/tester/data.go | 16 +++++++- 5 files changed, 100 insertions(+), 5 deletions(-) diff --git a/go.mod b/go.mod index ba515ab0..4ae4d394 100644 --- a/go.mod +++ b/go.mod @@ -9,5 +9,6 @@ require ( github.com/pkg/errors v0.9.1 github.com/spf13/cobra v1.3.0 github.com/stretchr/testify v1.7.0 + go.uber.org/zap v1.21.0 golang.org/x/sync v0.0.0-20210220032951-036812b2e83c ) diff --git a/go.sum b/go.sum index 3b1265fa..3ba38c2a 100644 --- a/go.sum +++ b/go.sum @@ -104,6 +104,7 @@ github.com/aws/aws-sdk-go-v2/service/route53 v1.1.1/go.mod h1:rLiOUrPLW/Er5kRcQ7 github.com/aws/aws-sdk-go-v2/service/sso v1.1.1/go.mod h1:SuZJxklHxLAXgLTc1iFXbEWkXs7QRTQpCLGaKIprQW0= github.com/aws/aws-sdk-go-v2/service/sts v1.1.1/go.mod h1:Wi0EBZwiz/K44YliU0EKxqTCJGUfYTWXrrBwkq736bM= github.com/aws/smithy-go v1.1.0/go.mod h1:EzMw8dbp/YJL4A5/sbhGddag+NPT7q084agLbB9LgIw= +github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= @@ -649,11 +650,16 @@ go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E= go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= +go.uber.org/atomic v1.7.0 h1:ADUqmZGgLDDfbSL9ZmPxKTybcoEYHgpYfELNoN+7hsw= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= +go.uber.org/goleak v1.1.11/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= +go.uber.org/multierr v1.6.0 h1:y6IPFStTAIT5Ytl7/XYmHvzXQ7S3g/IeZW9hyZ5thw4= go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= go.uber.org/zap v1.9.1/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= go.uber.org/zap v1.17.0/go.mod h1:MXVU+bhUf/A7Xi2HNOnopQOrmycQ5Ih87HtOu4q5SSo= +go.uber.org/zap v1.21.0 h1:WefMeulhovoZ2sYXz7st6K0sLj7bBhpiFaud4r4zST8= +go.uber.org/zap v1.21.0/go.mod h1:wjWOCqI0f2ZZrJF/UufIOkiC8ii6tm1iqIsLo76RfJw= golang.org/x/crypto v0.0.0-20170930174604-9419663f5a44/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= diff --git a/pkg/logger/logger.go b/pkg/logger/logger.go index 91b16ac8..9fd9cce2 100644 --- a/pkg/logger/logger.go +++ b/pkg/logger/logger.go @@ -21,6 +21,9 @@ import ( "os" "path" + "go.uber.org/zap" + "go.uber.org/zap/zapcore" + "github.com/coinbase/rosetta-cli/pkg/results" "github.com/coinbase/rosetta-sdk-go/parser" @@ -33,6 +36,8 @@ import ( var _ statefulsyncer.Logger = (*Logger)(nil) +type CheckType string + const ( // blockStreamFile contains the stream of processed // blocks and whether they were added or removed. @@ -58,6 +63,11 @@ const ( // removeEvent is printed in a stream // when an event is orphaned. removeEvent = "Remove" + + // Construction identifies construction check + Construction CheckType = "construction" + // Data identifies data check + Data CheckType = "data" ) // Logger contains all logic to record validator output @@ -71,6 +81,8 @@ type Logger struct { lastStatsMessage string lastProgressMessage string + + zapLogger *zap.Logger } // NewLogger constructs a new Logger. @@ -80,14 +92,43 @@ func NewLogger( logTransactions bool, logBalanceChanges bool, logReconciliation bool, -) *Logger { + checkType CheckType, + network *types.NetworkIdentifier, + fields ...zap.Field, +) (*Logger, error) { + zapLogger, err := buildZapLogger(checkType, network, fields...) + if err != nil { + return nil, err + } return &Logger{ logDir: logDir, logBlocks: logBlocks, logTransactions: logTransactions, logBalanceChanges: logBalanceChanges, logReconciliation: logReconciliation, + zapLogger: zapLogger, + }, nil +} + +func buildZapLogger( + checkType CheckType, + network *types.NetworkIdentifier, + fields ...zap.Field, +) (*zap.Logger, error) { + config := zap.NewDevelopmentConfig() + config.EncoderConfig.EncodeLevel = zapcore.CapitalColorLevelEncoder + + baseSlice := []zap.Field { + zap.String("blockchain", network.Blockchain), + zap.String("network", network.Network), + zap.String("check_type", string(checkType)), } + mergedSlice := append(baseSlice, fields...) + + zapLogger, err := config.Build( + zap.Fields(mergedSlice...), + ) + return zapLogger, err } // LogDataStatus logs results.CheckDataStatus. @@ -276,7 +317,7 @@ func (l *Logger) TransactionStream( block.BlockIdentifier.Index, block.BlockIdentifier.Hash, ) - + fmt.Print(transactionString) _, err = f.WriteString(transactionString) @@ -471,6 +512,36 @@ func (l *Logger) ReconcileFailureStream( return nil } +// Info logs at Info level +func (l *Logger) Info(msg string, fields ...zap.Field) { + l.zapLogger.Info(msg, fields...) +} + +// Debug logs at Debug level +func (l *Logger) Debug(msg string, fields ...zap.Field) { + l.zapLogger.Debug(msg, fields...) +} + +// Error logs at Error level +func (l *Logger) Error(msg string, fields ...zap.Field) { + l.zapLogger.Error(msg, fields...) +} + +// Warn logs at Warn level +func (l *Logger) Warn(msg string, fields ...zap.Field) { + l.zapLogger.Warn(msg, fields...) +} + +// Panic logs at Panic level +func (l *Logger) Panic(msg string, fields ...zap.Field) { + l.zapLogger.Panic(msg, fields...) +} + +// Fatal logs at Fatal level +func (l *Logger) Fatal(msg string, fields ...zap.Field) { + l.zapLogger.Fatal(msg, fields...) +} + // Helper function to close log file func closeFile(f *os.File) { err := f.Close() diff --git a/pkg/tester/construction.go b/pkg/tester/construction.go index 9f997d7a..653b3570 100644 --- a/pkg/tester/construction.go +++ b/pkg/tester/construction.go @@ -113,13 +113,18 @@ func InitializeConstruction( } counterStorage := modules.NewCounterStorage(localStore) - logger := logger.NewLogger( + logger, err := logger.NewLogger( dataPath, false, false, false, false, + logger.Construction, + network, ) + if err != nil { + return nil, fmt.Errorf("unable to initialize logger with error: %s", err.Error()) + } blockStorage := modules.NewBlockStorage(localStore, config.SerialBlockWorkers) keyStorage := modules.NewKeyStorage(localStore) diff --git a/pkg/tester/data.go b/pkg/tester/data.go index a24c67ed..382c644f 100644 --- a/pkg/tester/data.go +++ b/pkg/tester/data.go @@ -181,13 +181,19 @@ func InitializeData( blockStorage := modules.NewBlockStorage(localStore, config.SerialBlockWorkers) balanceStorage := modules.NewBalanceStorage(localStore) - logger := logger.NewLogger( + logger, err := logger.NewLogger( dataPath, config.Data.LogBlocks, config.Data.LogTransactions, config.Data.LogBalanceChanges, config.Data.LogReconciliations, + logger.Data, + network, ) + if err != nil { + log.Fatalf(fmt.Sprintf("unable to initialize logger with error: %s", err.Error())) + return nil + } var forceInactiveReconciliation bool reconcilerHelper := processor.NewReconcilerHelper( @@ -1018,14 +1024,20 @@ func (t *DataTester) recursiveOpSearch( blockStorage := modules.NewBlockStorage(localStore, t.config.SerialBlockWorkers) balanceStorage := modules.NewBalanceStorage(localStore) - logger := logger.NewLogger( + logger, err := logger.NewLogger( tmpDir, false, false, false, false, + logger.Data, + t.network, ) + if err != nil { + return nil, fmt.Errorf("unable to initialize logger with error: %s", err.Error()) + } + t.forceInactiveReconciliation = types.Bool(false) reconcilerHelper := processor.NewReconcilerHelper( t.config, From 4ded77b81d6268b32c9d355143a40c4e9caf6e43 Mon Sep 17 00:00:00 2001 From: racbc Date: Mon, 28 Feb 2022 11:47:36 -0600 Subject: [PATCH 292/376] Update README and CONTRIBUTING files information (#268) * Update README information * Update copyright year to current year * Update README to match new repo template * Update s/discrepency/discrepancy misspelling in several files * Update CONTRIBUTING file with new template * Update CONTRIBUTING file info after feedback * Post-review updates to CONTRIBUTING.md file --- CONTRIBUTING.md | 93 ++++-- README.md | 574 +++--------------------------------- pkg/results/data_results.go | 2 +- 3 files changed, 111 insertions(+), 558 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 217d1ede..23a0341a 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -2,52 +2,87 @@ ## Code of Conduct -All interactions with this project follow our [Code of Conduct][code-of-conduct]. -By participating, you are expected to honor this code. Violators can be banned -from further participation in this project, or potentially all Coinbase projects. +All interactions with this project follow our [Code of Conduct](https://github.com/coinbase/code-of-conduct). By participating, you are expected to honor this code. Violators can be banned from further participation in this project, or potentially all Coinbase projects. -[code-of-conduct]: https://github.com/coinbase/code-of-conduct +## How to Contribute -## Bug Reports +You can contribute to this repository by asking questions, providing feedback, and reporting issues. -* Ensure your issue [has not already been reported][1]. It may already be fixed! -* Include the steps you carried out to produce the problem. -* Include the behavior you observed along with the behavior you expected, and - why you expected it. -* Include any relevant stack traces or debugging output. +### Asking Questions -## Feature Requests +Submit your questions via the [Rosetta Community boards][13]. -We welcome feedback with or without pull requests. If you have an idea for how -to improve the project, great! All we ask is that you take the time to write a -clear and concise explanation of what need you are trying to solve. If you have -thoughts on _how_ it can be solved, include those too! +### Providing Feedback + +You can also use the [Rosetta Community boards][13] to provide feedback. + +### Reporting Issues + +You can report issues by submitting bug reports, feature requests, or pull requests via GitHub. You **must** submit [security issues](#security-issues) and [support requests](#support-requests) through the links provided. + +#### Bug Reports + +Before filing a bug report, ensure that your issue [has not already been reported][1]. It may already be fixed! + +If your bug hasn’t been fixed, follow these steps to file a bug report: + +1. [Open an issue in GitHub][10]. +2. Add a title for your bug report. It should briefly describe the problem. +3. Follow the template that appears in the Write text box. This is the best way to describe the bug. +4. Click _Submit new issue_ to finish filing the bug report. + +#### Feature Requests + +We welcome feedback with or without pull requests. If you have an idea for how to improve the project, great! All we ask is that you take the time to write a clear and concise explanation of the need you are trying to solve. If you have thoughts on _how_ it can be solved, include those too! + +To submit a feature request, follow these steps: + +1. [Open an issue in GitHub][10]. +2. Add a title for your feature request. It should briefly describe your requested feature. +3. Follow the template that appears in the Write text box. This is the best way to explain your request. Be clear and concise in your responses. +4. Click _Submit new issue_ to submit the feature request. The best way to see a feature added, however, is to submit a pull request. -## Pull Requests +#### Pull Requests + +Before creating your pull request, it's usually worth asking whether the code you're planning on writing will be considered for merging. You can do this by [opening an issue][1] and asking. It may also help give the maintainers context for when the time comes to review your code. -* Before creating your pull request, it's usually worth asking if the code - you're planning on writing will actually be considered for merging. You can - do this by [opening an issue][1] and asking. It may also help give the - maintainers context for when the time comes to review your code. +Ensure that your [commit messages are well-written][2]. This can double as your pull request message, so it pays to take the time to write a clear message. -* Ensure your [commit messages are well-written][2]. This can double as your - pull request message, so it pays to take the time to write a clear message. +Additionally, make sure that you have written unit tests for your changes. If you're unsure as to what to test, don't hesitate to [open an issue][1] and ask! -* Add tests for your feature. You should be able to look at other tests for - examples. If you're unsure, don't hesitate to [open an issue][1] and ask! +To submit your pull request, follow these steps: -* Submit your pull request! +1. Follow these instructions on how to [open a pull request in GitHub][11]. +2. Click _Create pull request_ to submit your pull request. -## Support Requests +Once you submit your pull request, a reviewer will revise it, and either approve it or offer suggestions. -For security reasons, any communication referencing support tickets for Coinbase -products will be ignored. The request will have its content redacted and will -be locked to prevent further discussion. +#### Security Issues + +You can send a report through Coinbase's [H1 program][12]. Check out the [Security][14] tab for more information. + +#### Support Requests All support requests must be made via [our support team][3]. +**For security reasons, any communication referencing support tickets for Coinbase products will be ignored.** The request will have its content redacted and will be locked to prevent further discussion. + +© 2022 Coinbase + + [1]: https://github.com/coinbase/rosetta-cli/issues [2]: https://chris.beams.io/posts/git-commit/#seven-rules [3]: https://support.coinbase.com/customer/en/portal/articles/2288496-how-can-i-contact-coinbase-support- + +[5]: https://github.com/coinbase/rosetta-cli/issues/new/choose +[6]: https://github.com/coinbase/rosetta-cli/issues/new?assignees=&labels=bug&template=bug_report.md&title= +[7]: https://github.com/coinbase/rosetta-cli/issues/new?assignees=&labels=enhancement&template=feature_request.md&title= +[8]: https://github.com/coinbase/rosetta-cli/pulls +[9]: https://github.com/coinbase/rosetta-cli/compare +[10]: https://docs.github.com/en/desktop/contributing-and-collaborating-using-github-desktop/working-with-your-remote-repository-on-github-or-github-enterprise/creating-an-issue-or-pull-request#creating-an-issue +[11]: https://docs.github.com/en/desktop/contributing-and-collaborating-using-github-desktop/working-with-your-remote-repository-on-github-or-github-enterprise/creating-an-issue-or-pull-request#creating-a-pull-request +[12]: https://hackerone.com/coinbase +[13]: https://community.rosetta-api.org +[14]: https://github.com/coinbase/rosetta-cli/security diff --git a/README.md b/README.md index 20bb7f5e..c3add1ef 100644 --- a/README.md +++ b/README.md @@ -16,569 +16,75 @@ CLI to validate the correctness of Rosetta API implementations

-## Overview -The `rosetta-cli` is used by developers to test the correctness of their Rosetta -API implementations. The CLI also provides the ability to look up block contents -and account balances. +The `rosetta-cli` tool is used by developers to test the correctness of their Rosetta API implementations. The CLI also provides the ability to look up block contents and account balances. -## Documentation -Before diving into the CLI, we recommend taking a look at the Rosetta API Docs: - -* [Overview](https://www.rosetta-api.org/docs/welcome.html) -* [Data API](https://www.rosetta-api.org/docs/data_api_introduction.html) -* [Construction API](https://www.rosetta-api.org/docs/construction_api_introduction.html) +## Installation -## Install To download a binary for the latest release, run: ``` curl -sSfL https://raw.githubusercontent.com/coinbase/rosetta-cli/master/scripts/install.sh | sh -s ``` -The binary will be installed inside the `./bin` directory (relative to where the install command was run). +The binary will be installed inside the `./bin` directory (relative to the directory where you ran the installation command). _Downloading binaries from the Github UI will cause permission errors on Mac._ -### Installing in Custom Location +### Installing in a Custom Location To download the binary into a specific directory, run: ``` curl -sSfL https://raw.githubusercontent.com/coinbase/rosetta-cli/master/scripts/install.sh | sh -s -- -b ``` -## Usage -``` -CLI for the Rosetta API - -Usage: - rosetta-cli [command] - -Available Commands: - check:construction Check the correctness of a Rosetta Construction API Implementation - check:data Check the correctness of a Rosetta Data API Implementation - configuration:create Create a default configuration file at the provided path - configuration:validate Ensure a configuration file at the provided path is formatted correctly - help Help about any command - utils:asserter-configuration Generate a static configuration file for the Asserter - utils:train-zstd Generate a zstd dictionary for enhanced compression performance - version Print rosetta-cli version - view:balance View an account balance - view:block View a block - view:networks View all network statuses - -Flags: - --block-profile string Save the pprof block profile in the specified file - --configuration-file string Configuration file that provides connection and test settings. - If you would like to generate a starter configuration file (populated - with the defaults), run rosetta-cli configuration:create. - - Any fields not populated in the configuration file will be populated with - default values. - --cpu-profile string Save the pprof cpu profile in the specified file - -h, --help help for rosetta-cli - --mem-profile string Save the pprof mem profile in the specified file - -Use "rosetta-cli [command] --help" for more information about a command. -``` - -### Configuration -All `rosetta-cli` parameters are populated from a configuration file (`--configuration-file`) -provided at runtime. If a configuration file is not provided, the default -configuration is used. This default configuration can be viewed -[here](examples/configuration/default.json). Note, there is no default -configuration for running `check:construction` as this is very network-specific. -You can view a full list of all configuration options [here](https://pkg.go.dev/github.com/coinbase/rosetta-cli/configuration). - -In the `examples/configuration` directory, you can find examples configuration -files for running tests against a Bitcoin Rosetta implementation -([config](https://github.com/coinbase/rosetta-bitcoin/tree/master/rosetta-cli-conf)) and an Ethereum Rosetta -implementation ([config](https://github.com/coinbase/rosetta-ethereum/tree/master/rosetta-cli-conf)). - -#### Using environment variables - -It's possible to set the configuration file path using an environment variable instead -of using a CLI flag. See an example below: - -```bash -ROSETTA_CONFIGURATION_FILE=/path/to/cli/config rosetta -``` - -CLI flags take precedence over environment variables. - -#### Writing check:construction Tests -The new Construction API testing framework (first released in `rosetta-cli@v0.5.0`) uses -a new design pattern to allow for complex transaction construction orchestration. -You can read more about the design goals [here](https://community.rosetta-api.org/t/feedback-request-automated-construction-api-testing-improvements/146). - -Most teams write their Construction API tests using the -[Rosetta Constructor DSL](https://github.com/coinbase/rosetta-sdk-go/tree/master/constructor/dsl). -We have examples of a DSL files written for [UTXO-based chains](https://github.com/coinbase/rosetta-bitcoin/blob/master/rosetta-cli-conf/testnet/bitcoin.ros) -and [account-based chains](https://github.com/coinbase/rosetta-ethereum/blob/master/rosetta-cli-conf/testnet/ethereum.ros). - -##### Terminology -When first learning about a new topic, it is often useful to understand the -hierarchy of concerns. In the automated Construction API tester, this -"hierarchy" is as follows: -```text -Workflows -> Jobs - Scenarios - Actions -``` - -`Workflows` contain collections of `Scenarios` to execute. `Scenarios` are -executed atomically in database transactions (rolled back if execution fails) -and culminate in an optional broadcast. This means that a single `Workflow` -could contain multiple broadcasts (which can be useful for orchestrating -staking-related transactions that affect a single account). - -To perform a `Workflow`, we create a `Job`. This `Job` has a unique identifier -and stores state for all `Scenarios` in the `Workflow`. State is shared across -an entire `Job` so `Actions` in a `Scenario` can access the output of `Actions` -in other `Scenarios`. The syntax for accessing this shared state can be found -[here](https://github.com/tidwall/gjson/blob/master/SYNTAX.md). - -`Actions` are discrete operations that can be performed in the context of a -`Scenario`. A full list of all `Actions` that can be performed can be found -[here](https://pkg.go.dev/github.com/coinbase/rosetta-sdk-go/constructor/job#ActionType). - -If you have suggestions for more actions, please -[open an issue in `rosetta-sdk-go`](https://github.com/coinbase/rosetta-sdk-go/issues)! - -##### Workflows -To use the automated Construction API tester (without prefunded accounts), -you must implement 2 required `Workflows`: -* `create_account` -* `request_funds` - -_If you don't implement these 2 `Workflows`, processing could stall._ - -Please note that `create_account` can contain a transaction broadcast if -on-chain origination is required for new accounts on your blockchain. - -If you plan to run the automated Construction API tester in CI, you may wish to -provide [`prefunded accounts`](https://pkg.go.dev/github.com/coinbase/rosetta-cli/configuration#ConstructionConfiguration) -when running the tester (otherwise you would need to manually fund generated -accounts). - -Optionally, you can also provide a `return_funds` workflow that will be invoked -when exiting `check:construction`. This can be useful in CI when you want to return -all funds to a single accout or faucet (instead of black-holing them in all the addresses -created during testing). - -##### Broadcast Invocation -If you'd like to broadcast a transaction at the end of a `Scenario`, -you must populate the following fields: -* `.network` -* `.operations` -* `.confirmation_depth` (allows for stake-related transactions to complete before marking as a success) - -Optionally, you can populate the following field: -* `.preprocess_metadata` - -Once a transaction is confirmed on-chain (after the provided -`.confirmation_depth`, it is stored by the tester at -`.transaction` for access by other `Scenarios` in the same `Job`. - -##### Dry Runs -In UTXO-based blockchains, it may be necessary to amend the `operations` stored -in `.operations` based on the `suggested_fee` returned in -`/construction/metadata`. The automated Construction API tester supports -running a "dry run" of a transaction broadcast if you set the follow field: -* `.dry_run = true` - -The suggested fee will then be stored as `.suggested_fee` for use by -other `Scenarios` in the same `Job`. You can find an example of this in the -Bitcoin [config](examples/configuration/bitcoin.json). - -*If this field is not populated or set to `false`, the transaction -will be constructed, signed, and broadcast.* - -#### End Conditions -When running the `rosetta-cli` in a CI job, it is usually desired to exit -when certain conditions are met (or before then with an exit code of 1). We -provide this functionality through the use of "end conditions" which can be -specified in your configuration file. - -##### check:data -A full list of `check:data` end conditions can be found [here](https://pkg.go.dev/github.com/coinbase/rosetta-cli/configuration#DataEndConditions). -If any end condition is satisifed, we will exit and output the -results in `results_output_file` (if it is populated). - -##### check:construction -The `check:construction` end condition is a map of -workflow:count that indicates how many of each workflow -should be performed before `check:construction` should stop. -For example, `{"create_account": 5}` indicates that 5 `create_account` -workflows should be performed before stopping. - -Unlike `check:data`, all `check:construction` end conditions -must be satisifed before the `rosetta-cli` will exit. - -#### Disable Complex Checks -If you are just getting started with your implementation, you may want -to disable balance tracking (did any address balance go below zero?) and -reconciliation (does the balance I calculated match the balance returned -by the `/account/balance` endpoint?). Take a look at the -[simple configuration](examples/configuration/simple.json) for an example of -how to do this. - -#### Status Codes -If there are no issues found while running `check`, it will exit with a `0` status code. -If there are any issues, it will exit with a `1` status code. It can be useful -to run this command as an integration test for any changes to your implementation. - -### Commands -#### version -``` -Print rosetta-cli version - -Usage: - rosetta-cli version [flags] - -Flags: - -h, --help help for version - -Global Flags: - --block-profile string Save the pprof block profile in the specified file - --configuration-file string Configuration file that provides connection and test settings. - If you would like to generate a starter configuration file (populated - with the defaults), run rosetta-cli configuration:create. - - Any fields not populated in the configuration file will be populated with - default values. - --cpu-profile string Save the pprof cpu profile in the specified file - --mem-profile string Save the pprof mem profile in the specified file -``` - -#### check:data -``` -Check all server responses are properly constructed, that -there are no duplicate blocks and transactions, that blocks can be processed -from genesis to the current block (re-orgs handled automatically), and that -computed balance changes are equal to balance changes reported by the node. - -When re-running this command, it will start where it left off if you specify -some data directory. Otherwise, it will create a new temporary directory and start -again from the genesis block. If you want to discard some number of blocks -populate the start_index filed in the configuration file with some block index. -Starting from a given index can be useful to debug a small range of blocks for -issues but it is highly recommended you sync from start to finish to ensure -all correctness checks are performed. - -By default, account balances are looked up at specific heights (instead of -only at the current block). If your node does not support this functionality, -you can disable historical balance lookups in your configuration file. This will -make reconciliation much less efficient but it will still work. - -If check fails due to an INACTIVE reconciliation error (balance changed without -any corresponding operation), the cli will automatically try to find the block -missing an operation. If historical balance disabled is true, this automatic -debugging tool does not work. - -To debug an INACTIVE account reconciliation error without historical balance lookup, -set the interesting accounts to the path of a JSON file containing -accounts that will be actively checked for balance changes at each block. This -will return an error at the block where a balance change occurred with no -corresponding operations. - -If your blockchain has a genesis allocation of funds and you set -historical balance disabled to true, you must provide an -absolute path to a JSON file containing initial balances with the -bootstrap balance config. You can look at the examples folder for an example -of what one of these files looks like. - -Usage: - rosetta-cli check:data [flags] - -Flags: - -h, --help help for check:data - -Global Flags: - --block-profile string Save the pprof block profile in the specified file - --configuration-file string Configuration file that provides connection and test settings. - If you would like to generate a starter configuration file (populated - with the defaults), run rosetta-cli configuration:create. - - Any fields not populated in the configuration file will be populated with - default values. - --cpu-profile string Save the pprof cpu profile in the specified file - --mem-profile string Save the pprof mem profile in the specified file -``` - -#### check:construction -``` -The check:construction command runs an automated test of a -Construction API implementation by creating and broadcasting transactions -on a blockchain. In short, this tool generates new addresses, requests -funds, constructs transactions, signs transactions, broadcasts transactions, -and confirms transactions land on-chain. At each phase, a series of tests -are run to ensure that intermediate representations are correct (i.e. does -an unsigned transaction return a superset of operations provided during -construction?). - -Check out the https://github.com/coinbase/rosetta-cli/tree/master/examples -directory for examples of how to configure this test for Bitcoin and -Ethereum. - -Right now, this tool only supports transfer testing (for both account-based -and UTXO-based blockchains). However, we plan to add support for testing -arbitrary scenarios (i.e. staking, governance). - -Usage: - rosetta-cli check:construction [flags] - -Flags: - -h, --help help for check:construction - -Global Flags: - --block-profile string Save the pprof block profile in the specified file - --configuration-file string Configuration file that provides connection and test settings. - If you would like to generate a starter configuration file (populated - with the defaults), run rosetta-cli configuration:create. - - Any fields not populated in the configuration file will be populated with - default values. - --cpu-profile string Save the pprof cpu profile in the specified file - --mem-profile string Save the pprof mem profile in the specified file -``` - -#### configuration:create -``` -Create a default configuration file at the provided path - -Usage: - rosetta-cli configuration:create [flags] - -Flags: - -h, --help help for configuration:create - -Global Flags: - --block-profile string Save the pprof block profile in the specified file - --configuration-file string Configuration file that provides connection and test settings. - If you would like to generate a starter configuration file (populated - with the defaults), run rosetta-cli configuration:create. +## Updates and Releases - Any fields not populated in the configuration file will be populated with - default values. - --cpu-profile string Save the pprof cpu profile in the specified file - --mem-profile string Save the pprof mem profile in the specified file -``` - -#### configuration:validate -``` -Validate the correctness of a configuration file at the provided path +We recommend that you continually update your installation to the latest release as soon as possible. The latest release notes are available in our [Community](https://community.rosetta-api.org) board under the [Release](https://community.rosetta-api.org/c/releases/13) category. -Usage: - rosetta-cli configuration:validate [flags] +You can also view releases and change log information in the [Releases](https://github.com/coinbase/rosetta-cli/releases) section of this repository. -Flags: - -h, --help help for configuration:validate - -Global Flags: - --configuration-file string Configuration file that provides connection and test settings. - If you would like to generate a starter configuration file (populated - with the defaults), run rosetta-cli configuration:create. +## Documentation - Any fields not populated in the configuration file will be populated with - default values. -``` +You can find the Rosetta API documentation at [rosetta-api.org](https://www.rosetta-api.org/docs/welcome.html) -#### view:networks -``` -While debugging a Data API implementation, it can be very -useful to view network(s) status. This command fetches the network -status from all available networks and prints it to the terminal. +For more information on the rosetta-cli tool, read our [The rosetta-cli tool](https://www.rosetta-api.org/docs/rosetta_cli.html) documentation. -If this command errors, it is likely because the /network/* endpoints are -not formatted correctly. +For more information on how to test your implementation file with the `rosetta-cli` tool, read our [How to Test Your Rosetta Implementation](https://www.rosetta-api.org/docs/rosetta_test.html) documentation. -Usage: - rosetta-cli view:networks [flags] +Our documentation is divided into the following sections: -Flags: - -h, --help help for view:networks +* [Product Overview](https://www.rosetta-api.org/docs/welcome.html) +* [Getting Started](https://www.rosetta-api.org/docs/getting_started.html) +* [Rosetta API Spec](https://www.rosetta-api.org/docs/Reference.html) +* [Testing](https://www.rosetta-api.org/docs/rosetta_cli.html) +* [Best Practices](https://www.rosetta-api.org/docs/node_deployment.html) +* [Repositories](https://www.rosetta-api.org/docs/rosetta_specifications.html) -Global Flags: - --block-profile string Save the pprof block profile in the specified file - --configuration-file string Configuration file that provides connection and test settings. - If you would like to generate a starter configuration file (populated - with the defaults), run rosetta-cli configuration:create. +## Contributing - Any fields not populated in the configuration file will be populated with - default values. - --cpu-profile string Save the pprof cpu profile in the specified file - --mem-profile string Save the pprof mem profile in the specified file -``` +You may contribute to the `rosetta-cli` project in various ways: -#### view:balance -``` -While debugging, it is often useful to inspect the state -of an account at a certain block. This command allows you to look up -any account by providing a JSON representation of a types.AccountIdentifier -(and optionally a height to perform the query). - -For example, you could run view:balance '{"address":"interesting address"}' 1000 -to lookup the balance of an interesting address at block 1000. Allowing the -address to specified as JSON allows for querying by SubAccountIdentifier. - -Usage: - rosetta-cli view:balance [flags] - -Flags: - -h, --help help for view:balance - -Global Flags: - --block-profile string Save the pprof block profile in the specified file - --configuration-file string Configuration file that provides connection and test settings. - If you would like to generate a starter configuration file (populated - with the defaults), run rosetta-cli configuration:create. - - Any fields not populated in the configuration file will be populated with - default values. - --cpu-profile string Save the pprof cpu profile in the specified file - --mem-profile string Save the pprof mem profile in the specified file -``` +* [Asking Questions](CONTRIBUTING.md/#asking-questions) +* [Providing Feedback](CONTRIBUTING.md/#providing-feedback) +* [Reporting Issues](CONTRIBUTING.md/#reporting-issues) -#### view:block -``` -While debugging a Data API implementation, it can be very -useful to inspect block contents. This command allows you to fetch any -block by index to inspect its contents. It uses the -fetcher (https://github.com/coinbase/rosetta-sdk-go/tree/master/fetcher) package -to automatically get all transactions in the block and assert the format -of the block is correct before printing. - -If this command errors, it is likely because the block you are trying to -fetch is formatted incorrectly. - -Usage: - rosetta-cli view:block [flags] - -Flags: - -h, --help help for view:block - --only-changes Only print balance changes for accounts in the block - -Global Flags: - --block-profile string Save the pprof block profile in the specified file - --configuration-file string Configuration file that provides connection and test settings. - If you would like to generate a starter configuration file (populated - with the defaults), run rosetta-cli configuration:create. - - Any fields not populated in the configuration file will be populated with - default values. - --cpu-profile string Save the pprof cpu profile in the specified file - --mem-profile string Save the pprof mem profile in the specified file -``` +Read our [Contributing](CONTRIBUTING.MD) documentation for more information. -#### utils:asserter-configuration -``` -In production deployments, it is useful to initialize the response -Asserter (https://github.com/coinbase/rosetta-sdk-go/tree/master/asserter) using -a static configuration instead of intializing a configuration dynamically -from the node. This allows a client to error on new types/statuses that may -have been added in an update instead of silently erroring. - -To use this command, simply provide an absolute path as the argument for where -the configuration file should be saved (in JSON). - -Usage: - rosetta-cli utils:asserter-configuration [flags] - -Flags: - -h, --help help for utils:asserter-configuration - -Global Flags: - --block-profile string Save the pprof block profile in the specified file - --configuration-file string Configuration file that provides connection and test settings. - If you would like to generate a starter configuration file (populated - with the defaults), run rosetta-cli configuration:create. - - Any fields not populated in the configuration file will be populated with - default values. - --cpu-profile string Save the pprof cpu profile in the specified file - --mem-profile string Save the pprof mem profile in the specified file -``` +## rosetta-cli Tool Development -#### utils:train-zstd -``` -Zstandard (https://github.com/facebook/zstd) is used by -rosetta-sdk-go/storage to compress data stored to disk. It is possible -to improve compression performance by training a dictionary on a particular -storage namespace. This command runs this training and outputs a dictionary -that can be used with rosetta-sdk-go/storage. - -The arguments for this command are: - () - -You can learn more about dictionary compression on the Zstandard -website: https://github.com/facebook/zstd#the-case-for-small-data-compression - -Usage: - rosetta-cli utils:train-zstd [flags] - -Flags: - -h, --help help for utils:train-zstd - -Global Flags: - --block-profile string Save the pprof block profile in the specified file - --configuration-file string Configuration file that provides connection and test settings. - If you would like to generate a starter configuration file (populated - with the defaults), run rosetta-cli configuration:create. - - Any fields not populated in the configuration file will be populated with - default values. - --cpu-profile string Save the pprof cpu profile in the specified file - --mem-profile string Save the pprof mem profile in the specified file -``` +While working on improvements to this repository, we recommend that you use these commands to check your code: -## Correctness Checks -This tool performs a variety of correctness checks using the Rosetta Server. If -any correctness check fails, the CLI will exit and print out a detailed -message explaining the error. - -### Response Correctness -The validator uses the autogenerated [Go Client package](https://github.com/coinbase/rosetta-sdk-go) -to communicate with the Rosetta Server and assert that responses adhere -to the Rosetta interface specification. - -### Duplicate Hashes -The validator checks that a block hash or transaction hash is -never duplicated. - -### Non-negative Balances -The validator checks that an account balance does not go -negative from any operations. - -### Balance Reconciliation -#### Active Addresses -The CLI checks that the balance of an account computed by -its operations is equal to the balance of the account according -to the node. If this balance is not identical, the CLI will -exit. - -#### Inactive Addresses -The CLI randomly checks the balances of accounts that aren't -involved in any transactions. The balances of accounts could change -on the blockchain node without being included in an operation -returned by the Rosetta Data API. Recall that all balance-changing -operations should be returned by the Rosetta Data API. - -## Development * `make deps` to install dependencies * `make test` to run tests * `make lint` to lint the source code (included generated code) * `make release` to run one last check before opening a PR * `make compile version=RELEASE_TAG` to generate binaries -If you are developing on both rosetta-cli and rosetta-sdk-go, use [go.mod replace](https://golang.org/ref/mod#go-mod-file-replace) to reference local changes: +If you are developing on both the `rosetta-cli` and `rosetta-sdk-go` repositories, use [go.mod replace](https://golang.org/ref/mod#go-mod-file-replace) to reference local changes: ``` replace "github.com/coinbase/rosetta-sdk-go" v0.6.8 => "../rosetta-sdk-go" ``` ### Helper/Handler -Many of the packages use a `Helper/Handler` interface pattern to acquire -required information or to send events to some client implementation. An example -of this is in the `reconciler` package where a `Helper` is used to get -the account balance and the `Handler` is called to indicate whether the -reconciliation of an account was successful. +Many of the packages use a `Helper/Handler` interface pattern to acquire required information or to send events to some client implementation. An example of this is in the `reconciler` package where a `Helper` is used to get the account balance and the `Handler` is called to indicate whether the reconciliation of an account was successful. ### Repo Structure ``` @@ -590,16 +96,28 @@ pkg tester // test orchestrators ``` -## Troubleshoot -* While running `check:data` or `check:construction` option if you get the following error: +### Troubleshooting - ```dial tcp 127.0.0.1:8080: socket: too many open files: unable to sync to 1902533: unable to sync to 1902533``` +While running the `check:data` or `check:construction` option, if you get the following error: + +```dial tcp 127.0.0.1:8080: socket: too many open files: unable to sync to 1902533: unable to sync to 1902533``` - Please run `ulimit -n 10000` to increase the max concurrent opened file limit +Please run the `ulimit -n 10000` command to increase the max concurrent opened file limit. + +_Note: MacOS users, if you face `ulimit: setrlimit failed: invalid argument` error while setting `ulimit`, please run `sudo launchctl limit maxfiles 10000 200000` before setting the `ulimit`_ + +## Related Projects + +* [`rosetta-sdk-go`][https://github.com/coinbase/rosetta-sdk-go] — The `rosetta-sdk-go` SDK provides a collection of packages used for interaction with the Rosetta API specification. Much of the SDK code is generated from this, the [`rosetta-specifications`](https://github.com/coinbase/rosetta-specifications) repository. +* [`rosetta-specifications`](https://github.com/coinbase/rosetta-specifications) — The `rosetta-specifications` repository generates the SDK code in the [`rosetta-sdk-go`][https://github.com/coinbase/rosetta-sdk-go] repository. + +### Reference Implementations + +To help you with examples, we developed complete Rosetta API reference implementations for [Bitcoin](https://github.com/coinbase/rosetta-bitcoin) and [Ethereum](https://github.com/coinbase/rosetta-ethereum). Developers of Bitcoin-like or Ethereum-like blockchains may find it easier to fork these reference implementations than to write an implementation from scratch. - _Note: MacOS users, if you face `ulimit: setrlimit failed: invalid argument` error while setting `ulimit`, please run `sudo launchctl limit maxfiles 10000 200000` before setting the `ulimit`_ +You can also find community implementations for a variety of blockchains in the [rosetta-ecosystem][https://github.com/coinbase/rosetta-ecosystem] repository, and in the [ecosystem category](https://community.rosetta-api.org/c/ecosystem) of our community site. ## License This project is available open source under the terms of the [Apache 2.0 License](https://opensource.org/licenses/Apache-2.0). -© 2021 Coinbase +© 2022 Coinbase \ No newline at end of file diff --git a/pkg/results/data_results.go b/pkg/results/data_results.go index e6674094..1c2f98e6 100644 --- a/pkg/results/data_results.go +++ b/pkg/results/data_results.go @@ -498,7 +498,7 @@ func (c *CheckDataTests) Print() { table.Append( []string{ "Reconciliation", - "No balance discrepencies were found between computed and live balances", + "No balance discrepancies were found between computed and live balances", convertBool(c.Reconciliation), }, ) From 6d0efaf2e441c6d9e24136335154f8d1057af23b Mon Sep 17 00:00:00 2001 From: racbc Date: Mon, 28 Feb 2022 17:22:31 -0600 Subject: [PATCH 293/376] Formatting updates to README file (#278) --- README.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index c3add1ef..b40f2502 100644 --- a/README.md +++ b/README.md @@ -104,18 +104,18 @@ While running the `check:data` or `check:construction` option, if you get the fo Please run the `ulimit -n 10000` command to increase the max concurrent opened file limit. -_Note: MacOS users, if you face `ulimit: setrlimit failed: invalid argument` error while setting `ulimit`, please run `sudo launchctl limit maxfiles 10000 200000` before setting the `ulimit`_ +_Note: MacOS users, if you face `ulimit: setrlimit failed: invalid argument` error while setting `ulimit`, please run `sudo launchctl limit maxfiles 10000 200000` before setting the `ulimit`._ ## Related Projects -* [`rosetta-sdk-go`][https://github.com/coinbase/rosetta-sdk-go] — The `rosetta-sdk-go` SDK provides a collection of packages used for interaction with the Rosetta API specification. Much of the SDK code is generated from this, the [`rosetta-specifications`](https://github.com/coinbase/rosetta-specifications) repository. -* [`rosetta-specifications`](https://github.com/coinbase/rosetta-specifications) — The `rosetta-specifications` repository generates the SDK code in the [`rosetta-sdk-go`][https://github.com/coinbase/rosetta-sdk-go] repository. +* [`rosetta-sdk-go`](https://github.com/coinbase/rosetta-sdk-go) — The `rosetta-sdk-go` SDK provides a collection of packages used for interaction with the Rosetta API specification. Much of the SDK code is generated from this, the [`rosetta-specifications`](https://github.com/coinbase/rosetta-specifications) repository. +* [`rosetta-specifications`](https://github.com/coinbase/rosetta-specifications) — The `rosetta-specifications` repository generates the SDK code in the [`rosetta-sdk-go`](https://github.com/coinbase/rosetta-sdk-go) repository. ### Reference Implementations To help you with examples, we developed complete Rosetta API reference implementations for [Bitcoin](https://github.com/coinbase/rosetta-bitcoin) and [Ethereum](https://github.com/coinbase/rosetta-ethereum). Developers of Bitcoin-like or Ethereum-like blockchains may find it easier to fork these reference implementations than to write an implementation from scratch. -You can also find community implementations for a variety of blockchains in the [rosetta-ecosystem][https://github.com/coinbase/rosetta-ecosystem] repository, and in the [ecosystem category](https://community.rosetta-api.org/c/ecosystem) of our community site. +You can also find community implementations for a variety of blockchains in the [rosetta-ecosystem](https://github.com/coinbase/rosetta-ecosystem) repository, and in the [ecosystem category](https://community.rosetta-api.org/c/ecosystem) of our community site. ## License This project is available open source under the terms of the [Apache 2.0 License](https://opensource.org/licenses/Apache-2.0). From 5bde4a01e8146d2854ffa2a9780547ae26ddca5b Mon Sep 17 00:00:00 2001 From: racbc Date: Mon, 28 Feb 2022 18:18:22 -0600 Subject: [PATCH 294/376] Formatting updates to README file (#279) From 18013bd2628aa98d9cde87e5d10b3216c35d070c Mon Sep 17 00:00:00 2001 From: racbc Date: Thu, 3 Mar 2022 14:59:31 -0600 Subject: [PATCH 295/376] Update text in Issue Templates (#280) * Update text in Issue Templates * Add post-review edits to Issue Templates --- .github/ISSUE_TEMPLATE/bug_report.md | 7 ++++--- .github/ISSUE_TEMPLATE/feature_request.md | 9 +++++---- 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md index 4e098f8a..5b07f85f 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.md +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -8,12 +8,13 @@ assignees: '' --- **Describe the bug** -A clear and concise description of what the bug is. + **To Reproduce** -Steps to reproduce the behavior: + **Expected behavior** -A clear and concise description of what you expected to happen. + **Additional context** + diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md index 36014cde..735569fe 100644 --- a/.github/ISSUE_TEMPLATE/feature_request.md +++ b/.github/ISSUE_TEMPLATE/feature_request.md @@ -8,13 +8,14 @@ assignees: '' --- **Is your feature request related to a problem? Please describe.** -A clear and concise description of what the problem is. Ex. I'm always frustrated when [...] + **Describe the solution you'd like** -A clear and concise description of what you want to happen. + **Describe alternatives you've considered** -A clear and concise description of any alternative solutions or features you've considered. + **Additional context** -Add any other context or screenshots about the feature request here. + + From 927f044c552d43770fbf0e8df016568c26917fff Mon Sep 17 00:00:00 2001 From: racbc Date: Thu, 24 Mar 2022 20:34:07 -0500 Subject: [PATCH 296/376] Fix bad use of i.e. in check_construction.go (#292) --- cmd/check_construction.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cmd/check_construction.go b/cmd/check_construction.go index d4d24e26..2a04aa79 100644 --- a/cmd/check_construction.go +++ b/cmd/check_construction.go @@ -38,7 +38,7 @@ Construction API implementation by creating and broadcasting transactions on a blockchain. In short, this tool generates new addresses, requests funds, constructs transactions, signs transactions, broadcasts transactions, and confirms transactions land on-chain. At each phase, a series of tests -are run to ensure that intermediate representations are correct (i.e. does +are run to ensure that intermediate representations are correct (for example, does an unsigned transaction return a superset of operations provided during construction?). @@ -48,7 +48,7 @@ Ethereum. Right now, this tool only supports transfer testing (for both account-based and UTXO-based blockchains). However, we plan to add support for testing -arbitrary scenarios (i.e. staking, governance).`, +arbitrary scenarios (for example, staking and governance).`, RunE: runCheckConstructionCmd, } ) From ad9c2dfae650cbc9040f1ff9ab15bd58e0166512 Mon Sep 17 00:00:00 2001 From: Madhur Shrimal Date: Fri, 25 Mar 2022 17:28:01 +0530 Subject: [PATCH 297/376] update rosetta-sdk (#284) --- go.mod | 2 +- go.sum | 35 ++++++++++++++++++----------------- scripts/compile.sh | 4 ++-- 3 files changed, 21 insertions(+), 20 deletions(-) diff --git a/go.mod b/go.mod index 4ae4d394..f45be1fc 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/coinbase/rosetta-cli go 1.16 require ( - github.com/coinbase/rosetta-sdk-go v0.7.3 + github.com/coinbase/rosetta-sdk-go v0.7.4 github.com/fatih/color v1.13.0 github.com/olekukonko/tablewriter v0.0.5 github.com/pkg/errors v0.9.1 diff --git a/go.sum b/go.sum index 3ba38c2a..5a584983 100644 --- a/go.sum +++ b/go.sum @@ -104,6 +104,7 @@ github.com/aws/aws-sdk-go-v2/service/route53 v1.1.1/go.mod h1:rLiOUrPLW/Er5kRcQ7 github.com/aws/aws-sdk-go-v2/service/sso v1.1.1/go.mod h1:SuZJxklHxLAXgLTc1iFXbEWkXs7QRTQpCLGaKIprQW0= github.com/aws/aws-sdk-go-v2/service/sts v1.1.1/go.mod h1:Wi0EBZwiz/K44YliU0EKxqTCJGUfYTWXrrBwkq736bM= github.com/aws/smithy-go v1.1.0/go.mod h1:EzMw8dbp/YJL4A5/sbhGddag+NPT7q084agLbB9LgIw= +github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8= github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= @@ -158,10 +159,10 @@ github.com/cncf/xds/go v0.0.0-20210922020428-25de7278fc84/go.mod h1:eXthEFrGJvWH github.com/cncf/xds/go v0.0.0-20211001041855-01bcc9b48dfe/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cncf/xds/go v0.0.0-20211130200136-a8f946100490/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= -github.com/coinbase/kryptology v1.5.5 h1:iSB2AOGYBxvq+/4huDWVEnyCWWWrVBbb7yiCcLrE2rw= -github.com/coinbase/kryptology v1.5.5/go.mod h1:3zId6V3k/8v/o5gksjffbWMkH7Ik1UgeZVGTBvlpIxc= -github.com/coinbase/rosetta-sdk-go v0.7.3 h1:SnnJ5VJGZe6l4rYhyNha94SVUJinVwtZTPvtfgF2Xfw= -github.com/coinbase/rosetta-sdk-go v0.7.3/go.mod h1:PeO5OtybTPoddCrBrWaJS2vBICc2oLFoJkIIuRS/wPo= +github.com/coinbase/kryptology v1.8.0 h1:Aoq4gdTsJhSU3lNWsD5BWmFSz2pE0GlmrljaOxepdYY= +github.com/coinbase/kryptology v1.8.0/go.mod h1:RYXOAPdzOGUe3qlSFkMGn58i3xUA8hmxYHksuq+8ciI= +github.com/coinbase/rosetta-sdk-go v0.7.4 h1:O1Usxxa1Bg2qiOGcbWZHXtjvOAeTUjk/gkVxYLaPtiw= +github.com/coinbase/rosetta-sdk-go v0.7.4/go.mod h1:lt2lJX/HQ2stryuKkukCWMcX3T5fBU62UrdhuwsnVYg= github.com/consensys/bavard v0.1.8-0.20210406032232-f3452dc9b572/go.mod h1:Bpd0/3mZuaj6Sj+PqrmIquiOKy397AKGThQPaGzNXAQ= github.com/consensys/bavard v0.1.8-0.20210915155054-088da2f7f54a/go.mod h1:9ItSMtA/dXMAiL7BG6bqW2m3NdSEObYWoH223nGHukI= github.com/consensys/gnark-crypto v0.4.1-0.20210426202927-39ac3d4b3f1f/go.mod h1:815PAHg3wvysy0SyIqanF8gZ0Y1wjk/hrDHD/iT88+Q= @@ -182,7 +183,7 @@ github.com/davecgh/go-spew v0.0.0-20171005155431-ecdeabc65495/go.mod h1:J7Y8YcW2 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/deckarep/golang-set v0.0.0-20180603214616-504e848d77ea/go.mod h1:93vsz/8Wt4joVM7c2AVqh+YRMiUSc14yDtF28KmMOgQ= +github.com/deckarep/golang-set v1.8.0/go.mod h1:5nI87KwE7wgsBU1F4GKAw2Qod7p5kyS383rP6+o6qqo= github.com/decred/dcrd/lru v1.0.0/go.mod h1:mxKOwFd7lFjN2GZYsiz/ecgqR6kkYAl+0pz0tEMk218= github.com/deepmap/oapi-codegen v1.6.0/go.mod h1:ryDa9AgbELGeB+YEXE1dR53yAjHwFvE9iAUlWl9Al3M= github.com/deepmap/oapi-codegen v1.8.2/go.mod h1:YLgSKSDv/bZQB7N4ws6luhozi3cEdRktEqrX88CvjIw= @@ -216,8 +217,8 @@ github.com/envoyproxy/go-control-plane v0.9.10-0.20210907150352-cf90f659a021/go. github.com/envoyproxy/go-control-plane v0.10.1/go.mod h1:AY7fTTXNdv/aJ2O5jwpxAPOWUZ7hQAEvzN5Pf27BkQQ= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/envoyproxy/protoc-gen-validate v0.6.2/go.mod h1:2t7qjJNvHPx8IjnBOzl9E9/baC+qXE/TeeyBRzgJDws= -github.com/ethereum/go-ethereum v1.10.13 h1:DEYFP9zk+Gruf3ae1JOJVhNmxK28ee+sMELPLgYTXpA= -github.com/ethereum/go-ethereum v1.10.13/go.mod h1:W3yfrFyL9C1pHcwY5hmRHVDaorTiQxhYBkKyu5mEDHw= +github.com/ethereum/go-ethereum v1.10.16 h1:3oPrumn0bCW/idjcxMn5YYVCdK7VzJYIvwGZUGLEaoc= +github.com/ethereum/go-ethereum v1.10.16/go.mod h1:Anj6cxczl+AHy63o4X9O8yWNHuN5wMpfb8MAnHkWn7Y= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/fatih/color v1.9.0/go.mod h1:eQcE1qtQxscV5RaZvpXrrb8Drkc3/DdQ+uUYCNjL+zU= github.com/fatih/color v1.13.0 h1:8LOYc1KYPPmyKMuN8QV2DNRWNbLo6LZ0iLs8+mlH53w= @@ -339,7 +340,7 @@ github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORR github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= github.com/gorilla/websocket v1.4.1/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= -github.com/graph-gophers/graphql-go v0.0.0-20201113091052-beb923fada29/go.mod h1:9CQHMSxwO4MprSdzoIEobiHpoLtHm77vfxsvsIN5Vuc= +github.com/graph-gophers/graphql-go v1.3.0/go.mod h1:9CQHMSxwO4MprSdzoIEobiHpoLtHm77vfxsvsIN5Vuc= github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= github.com/gtank/merlin v0.1.1/go.mod h1:T86dnYJhcGOh5BjZFCJWTDeTK7XW8uE+E21Cy/bIQ+s= github.com/hashicorp/consul/api v1.11.0/go.mod h1:XjsvQN+RJGWI2TWy1/kqaE16HrR2J/FWgkYjdZQsX9M= @@ -395,7 +396,7 @@ github.com/influxdata/promql/v2 v2.12.0/go.mod h1:fxOPu+DY0bqCTCECchSRtWfc+0X19y github.com/influxdata/roaring v0.4.13-0.20180809181101-fc520f41fab6/go.mod h1:bSgUQ7q5ZLSO+bKBGqJiCBGAl+9DxyW63zLTujjUlOE= github.com/influxdata/tdigest v0.0.0-20181121200506-bf2b5ad3c0a9/go.mod h1:Js0mqiSBE6Ffsg94weZZ2c+v/ciT8QRHFOap7EKDrR0= github.com/influxdata/usage-client v0.0.0-20160829180054-6d3895376368/go.mod h1:Wbbw6tYNvwa5dlB6304Sd+82Z3f7PmVZHVKU637d4po= -github.com/jackpal/go-nat-pmp v1.0.2-0.20160603034137-1fa385a6f458/go.mod h1:QPH045xvCAeXUZOxsnwmrtiCoxIr9eob+4orBN1SBKc= +github.com/jackpal/go-nat-pmp v1.0.2/go.mod h1:QPH045xvCAeXUZOxsnwmrtiCoxIr9eob+4orBN1SBKc= github.com/jedisct1/go-minisign v0.0.0-20190909160543-45766022959e/go.mod h1:G1CVv03EnqU1wYL2dFwXxW2An0az9JTl/ZsqXQeBlkU= github.com/jessevdk/go-flags v0.0.0-20141203071132-1679536dcc89/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= @@ -413,7 +414,7 @@ github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfV github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/jung-kurt/gofpdf v1.0.3-0.20190309125859-24315acbbda5/go.mod h1:7Id9E/uU8ce6rXgefFLlgrJj/GYY22cpxn+r32jIOes= github.com/jwilder/encoding v0.0.0-20170811194829-b4e1701a28ef/go.mod h1:Ct9fl0F6iIOGgxJ5npU/IUOhOhqlVrGjyIZc8/MagT0= -github.com/karalabe/usb v0.0.0-20211005121534-4c5740d64559/go.mod h1:Od972xHfMJowv7NGVDiWVxk2zxnWgjLlJzE+F4F7AGU= +github.com/karalabe/usb v0.0.2/go.mod h1:Od972xHfMJowv7NGVDiWVxk2zxnWgjLlJzE+F4F7AGU= github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= @@ -499,8 +500,6 @@ github.com/naoina/go-stringutil v0.1.0/go.mod h1:XJ2SJL9jCtBh+P9q5btrd/Ylo8XwT/h github.com/naoina/toml v0.1.2-0.20170918210437-9fafd6967416/go.mod h1:NBIhNtsFMo3G2szEBne+bO4gS192HuIYRqfvOWb4i1E= github.com/neilotoole/errgroup v0.1.6 h1:PODGqPXdT5BC/zCYIMoTrwV+ujKcW+gBXM6Ye9Ve3R8= github.com/neilotoole/errgroup v0.1.6/go.mod h1:Q2nLGf+594h0CLBs/Mbg6qOr7GtqDK7C2S41udRnToE= -github.com/nixberg/chacha-rng-go v0.1.0 h1:2Y90hS8H/O+82yLWKD4MldkJM7Pzhezc7rT6EILvDNo= -github.com/nixberg/chacha-rng-go v0.1.0/go.mod h1:iPf1i6Vcwgoue86dblORobEXNCd1PFQiysFiImawfCM= github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= github.com/olekukonko/tablewriter v0.0.5 h1:P2Ga83D34wi1o9J6Wh1mRuqd4mF/x/lgBS7N7AbDhec= @@ -605,15 +604,15 @@ github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5Cc github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7/go.mod h1:q4W45IWZaF22tdD+VEXcAWRA037jwmWEB5VWYORlTpc= -github.com/tidwall/gjson v1.10.2/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= -github.com/tidwall/gjson v1.12.0 h1:61wEp/qfvFnqKH/WCI3M8HuRut+mHT6Mr82QrFmM2SY= -github.com/tidwall/gjson v1.12.0/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= +github.com/tidwall/gjson v1.12.1/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= +github.com/tidwall/gjson v1.14.0 h1:6aeJ0bzojgWLa82gDQHcx3S0Lr/O51I9bJ5nv6JFx5w= +github.com/tidwall/gjson v1.14.0/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA= github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM= github.com/tidwall/pretty v1.2.0 h1:RWIZEg2iJ8/g6fDDYzMpobmaoGh5OLl4AXtGUGPcqCs= github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= -github.com/tidwall/sjson v1.2.3 h1:5+deguEhHSEjmuICXZ21uSSsXotWMA0orU783+Z7Cp8= -github.com/tidwall/sjson v1.2.3/go.mod h1:5WdjKx3AQMvCJ4RG6/2UYT7dLrGvJUV1x4jdTAyGvZs= +github.com/tidwall/sjson v1.2.4 h1:cuiLzLnaMeBhRmEv00Lpk3tkYrcxpmbU81tAY4Dw0tc= +github.com/tidwall/sjson v1.2.4/go.mod h1:098SZ494YoMWPmMO6ct4dcFnqxwj9r/gF0Etp19pSNM= github.com/tinylib/msgp v1.0.2/go.mod h1:+d+yLhGm8mzTaHzB+wgMYrodPfmZrzkirds8fDWklFE= github.com/tklauser/go-sysconf v0.3.5/go.mod h1:MkWzOF4RMCshBAMXuhXJs64Rte09mITnppBXY/rYEFI= github.com/tklauser/numcpus v0.2.2/go.mod h1:x3qojaO3uyYt0i56EW/VUYs7uBvdl2fkfZFu0T9wgjM= @@ -652,6 +651,7 @@ go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqe go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.7.0 h1:ADUqmZGgLDDfbSL9ZmPxKTybcoEYHgpYfELNoN+7hsw= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= +go.uber.org/goleak v1.1.11 h1:wy28qYRKZgnJTxGxvye5/wgWr1EKjmUDGYox5mGlRlI= go.uber.org/goleak v1.1.11/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= go.uber.org/multierr v1.6.0 h1:y6IPFStTAIT5Ytl7/XYmHvzXQ7S3g/IeZW9hyZ5thw4= @@ -1134,6 +1134,7 @@ gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo= diff --git a/scripts/compile.sh b/scripts/compile.sh index 40cb9f41..8ce67c52 100755 --- a/scripts/compile.sh +++ b/scripts/compile.sh @@ -26,8 +26,8 @@ TARGETS="${MAC_TARGETS},${LINUX_TARGETS},${WINDOWS_TARGET}" xgo -go 1.16.3 --targets=${TARGETS} -out "bin/rosetta-cli-${VERSION}" .; # Rename some files -mv "bin/rosetta-cli-${VERSION}-darwin-10.12-amd64" "bin/rosetta-cli-${VERSION}-darwin-amd64" -mv "bin/rosetta-cli-${VERSION}-darwin-10.12-arm64" "bin/rosetta-cli-${VERSION}-darwin-arm64" +mv "bin/rosetta-cli-${VERSION}-darwin-10.16-amd64" "bin/rosetta-cli-${VERSION}-darwin-amd64" +mv "bin/rosetta-cli-${VERSION}-darwin-10.16-arm64" "bin/rosetta-cli-${VERSION}-darwin-arm64" mv "bin/rosetta-cli-${VERSION}-windows-4.0-amd64.exe" "bin/rosetta-cli-${VERSION}-windows-amd64" # Tar all files From dacb2494fde841def213ac08ae538a1171e0e00b Mon Sep 17 00:00:00 2001 From: Shi Cheng Date: Thu, 24 Mar 2022 10:28:03 -0400 Subject: [PATCH 298/376] ROSE-231: Fix 'insufficient fund' error for UTXO chain when required_coins is true --- go.mod | 2 +- go.sum | 35 ++++++++++++++++++----------------- pkg/tester/construction.go | 30 +++++++++++++++++++++++++++++- 3 files changed, 48 insertions(+), 19 deletions(-) diff --git a/go.mod b/go.mod index 4ae4d394..b20729ce 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/coinbase/rosetta-cli go 1.16 require ( - github.com/coinbase/rosetta-sdk-go v0.7.3 + github.com/coinbase/rosetta-sdk-go v0.7.6 github.com/fatih/color v1.13.0 github.com/olekukonko/tablewriter v0.0.5 github.com/pkg/errors v0.9.1 diff --git a/go.sum b/go.sum index 3ba38c2a..a6a70135 100644 --- a/go.sum +++ b/go.sum @@ -104,6 +104,7 @@ github.com/aws/aws-sdk-go-v2/service/route53 v1.1.1/go.mod h1:rLiOUrPLW/Er5kRcQ7 github.com/aws/aws-sdk-go-v2/service/sso v1.1.1/go.mod h1:SuZJxklHxLAXgLTc1iFXbEWkXs7QRTQpCLGaKIprQW0= github.com/aws/aws-sdk-go-v2/service/sts v1.1.1/go.mod h1:Wi0EBZwiz/K44YliU0EKxqTCJGUfYTWXrrBwkq736bM= github.com/aws/smithy-go v1.1.0/go.mod h1:EzMw8dbp/YJL4A5/sbhGddag+NPT7q084agLbB9LgIw= +github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8= github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= @@ -158,10 +159,10 @@ github.com/cncf/xds/go v0.0.0-20210922020428-25de7278fc84/go.mod h1:eXthEFrGJvWH github.com/cncf/xds/go v0.0.0-20211001041855-01bcc9b48dfe/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cncf/xds/go v0.0.0-20211130200136-a8f946100490/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= -github.com/coinbase/kryptology v1.5.5 h1:iSB2AOGYBxvq+/4huDWVEnyCWWWrVBbb7yiCcLrE2rw= -github.com/coinbase/kryptology v1.5.5/go.mod h1:3zId6V3k/8v/o5gksjffbWMkH7Ik1UgeZVGTBvlpIxc= -github.com/coinbase/rosetta-sdk-go v0.7.3 h1:SnnJ5VJGZe6l4rYhyNha94SVUJinVwtZTPvtfgF2Xfw= -github.com/coinbase/rosetta-sdk-go v0.7.3/go.mod h1:PeO5OtybTPoddCrBrWaJS2vBICc2oLFoJkIIuRS/wPo= +github.com/coinbase/kryptology v1.8.0 h1:Aoq4gdTsJhSU3lNWsD5BWmFSz2pE0GlmrljaOxepdYY= +github.com/coinbase/kryptology v1.8.0/go.mod h1:RYXOAPdzOGUe3qlSFkMGn58i3xUA8hmxYHksuq+8ciI= +github.com/coinbase/rosetta-sdk-go v0.7.6 h1:y3/Lxu9ojIsUKnhNc5UaZF/i0byfuNDWXLYMSKG+IBs= +github.com/coinbase/rosetta-sdk-go v0.7.6/go.mod h1:lt2lJX/HQ2stryuKkukCWMcX3T5fBU62UrdhuwsnVYg= github.com/consensys/bavard v0.1.8-0.20210406032232-f3452dc9b572/go.mod h1:Bpd0/3mZuaj6Sj+PqrmIquiOKy397AKGThQPaGzNXAQ= github.com/consensys/bavard v0.1.8-0.20210915155054-088da2f7f54a/go.mod h1:9ItSMtA/dXMAiL7BG6bqW2m3NdSEObYWoH223nGHukI= github.com/consensys/gnark-crypto v0.4.1-0.20210426202927-39ac3d4b3f1f/go.mod h1:815PAHg3wvysy0SyIqanF8gZ0Y1wjk/hrDHD/iT88+Q= @@ -182,7 +183,7 @@ github.com/davecgh/go-spew v0.0.0-20171005155431-ecdeabc65495/go.mod h1:J7Y8YcW2 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/deckarep/golang-set v0.0.0-20180603214616-504e848d77ea/go.mod h1:93vsz/8Wt4joVM7c2AVqh+YRMiUSc14yDtF28KmMOgQ= +github.com/deckarep/golang-set v1.8.0/go.mod h1:5nI87KwE7wgsBU1F4GKAw2Qod7p5kyS383rP6+o6qqo= github.com/decred/dcrd/lru v1.0.0/go.mod h1:mxKOwFd7lFjN2GZYsiz/ecgqR6kkYAl+0pz0tEMk218= github.com/deepmap/oapi-codegen v1.6.0/go.mod h1:ryDa9AgbELGeB+YEXE1dR53yAjHwFvE9iAUlWl9Al3M= github.com/deepmap/oapi-codegen v1.8.2/go.mod h1:YLgSKSDv/bZQB7N4ws6luhozi3cEdRktEqrX88CvjIw= @@ -216,8 +217,8 @@ github.com/envoyproxy/go-control-plane v0.9.10-0.20210907150352-cf90f659a021/go. github.com/envoyproxy/go-control-plane v0.10.1/go.mod h1:AY7fTTXNdv/aJ2O5jwpxAPOWUZ7hQAEvzN5Pf27BkQQ= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/envoyproxy/protoc-gen-validate v0.6.2/go.mod h1:2t7qjJNvHPx8IjnBOzl9E9/baC+qXE/TeeyBRzgJDws= -github.com/ethereum/go-ethereum v1.10.13 h1:DEYFP9zk+Gruf3ae1JOJVhNmxK28ee+sMELPLgYTXpA= -github.com/ethereum/go-ethereum v1.10.13/go.mod h1:W3yfrFyL9C1pHcwY5hmRHVDaorTiQxhYBkKyu5mEDHw= +github.com/ethereum/go-ethereum v1.10.16 h1:3oPrumn0bCW/idjcxMn5YYVCdK7VzJYIvwGZUGLEaoc= +github.com/ethereum/go-ethereum v1.10.16/go.mod h1:Anj6cxczl+AHy63o4X9O8yWNHuN5wMpfb8MAnHkWn7Y= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/fatih/color v1.9.0/go.mod h1:eQcE1qtQxscV5RaZvpXrrb8Drkc3/DdQ+uUYCNjL+zU= github.com/fatih/color v1.13.0 h1:8LOYc1KYPPmyKMuN8QV2DNRWNbLo6LZ0iLs8+mlH53w= @@ -339,7 +340,7 @@ github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORR github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= github.com/gorilla/websocket v1.4.1/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= -github.com/graph-gophers/graphql-go v0.0.0-20201113091052-beb923fada29/go.mod h1:9CQHMSxwO4MprSdzoIEobiHpoLtHm77vfxsvsIN5Vuc= +github.com/graph-gophers/graphql-go v1.3.0/go.mod h1:9CQHMSxwO4MprSdzoIEobiHpoLtHm77vfxsvsIN5Vuc= github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= github.com/gtank/merlin v0.1.1/go.mod h1:T86dnYJhcGOh5BjZFCJWTDeTK7XW8uE+E21Cy/bIQ+s= github.com/hashicorp/consul/api v1.11.0/go.mod h1:XjsvQN+RJGWI2TWy1/kqaE16HrR2J/FWgkYjdZQsX9M= @@ -395,7 +396,7 @@ github.com/influxdata/promql/v2 v2.12.0/go.mod h1:fxOPu+DY0bqCTCECchSRtWfc+0X19y github.com/influxdata/roaring v0.4.13-0.20180809181101-fc520f41fab6/go.mod h1:bSgUQ7q5ZLSO+bKBGqJiCBGAl+9DxyW63zLTujjUlOE= github.com/influxdata/tdigest v0.0.0-20181121200506-bf2b5ad3c0a9/go.mod h1:Js0mqiSBE6Ffsg94weZZ2c+v/ciT8QRHFOap7EKDrR0= github.com/influxdata/usage-client v0.0.0-20160829180054-6d3895376368/go.mod h1:Wbbw6tYNvwa5dlB6304Sd+82Z3f7PmVZHVKU637d4po= -github.com/jackpal/go-nat-pmp v1.0.2-0.20160603034137-1fa385a6f458/go.mod h1:QPH045xvCAeXUZOxsnwmrtiCoxIr9eob+4orBN1SBKc= +github.com/jackpal/go-nat-pmp v1.0.2/go.mod h1:QPH045xvCAeXUZOxsnwmrtiCoxIr9eob+4orBN1SBKc= github.com/jedisct1/go-minisign v0.0.0-20190909160543-45766022959e/go.mod h1:G1CVv03EnqU1wYL2dFwXxW2An0az9JTl/ZsqXQeBlkU= github.com/jessevdk/go-flags v0.0.0-20141203071132-1679536dcc89/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= @@ -413,7 +414,7 @@ github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfV github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/jung-kurt/gofpdf v1.0.3-0.20190309125859-24315acbbda5/go.mod h1:7Id9E/uU8ce6rXgefFLlgrJj/GYY22cpxn+r32jIOes= github.com/jwilder/encoding v0.0.0-20170811194829-b4e1701a28ef/go.mod h1:Ct9fl0F6iIOGgxJ5npU/IUOhOhqlVrGjyIZc8/MagT0= -github.com/karalabe/usb v0.0.0-20211005121534-4c5740d64559/go.mod h1:Od972xHfMJowv7NGVDiWVxk2zxnWgjLlJzE+F4F7AGU= +github.com/karalabe/usb v0.0.2/go.mod h1:Od972xHfMJowv7NGVDiWVxk2zxnWgjLlJzE+F4F7AGU= github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= @@ -499,8 +500,6 @@ github.com/naoina/go-stringutil v0.1.0/go.mod h1:XJ2SJL9jCtBh+P9q5btrd/Ylo8XwT/h github.com/naoina/toml v0.1.2-0.20170918210437-9fafd6967416/go.mod h1:NBIhNtsFMo3G2szEBne+bO4gS192HuIYRqfvOWb4i1E= github.com/neilotoole/errgroup v0.1.6 h1:PODGqPXdT5BC/zCYIMoTrwV+ujKcW+gBXM6Ye9Ve3R8= github.com/neilotoole/errgroup v0.1.6/go.mod h1:Q2nLGf+594h0CLBs/Mbg6qOr7GtqDK7C2S41udRnToE= -github.com/nixberg/chacha-rng-go v0.1.0 h1:2Y90hS8H/O+82yLWKD4MldkJM7Pzhezc7rT6EILvDNo= -github.com/nixberg/chacha-rng-go v0.1.0/go.mod h1:iPf1i6Vcwgoue86dblORobEXNCd1PFQiysFiImawfCM= github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= github.com/olekukonko/tablewriter v0.0.5 h1:P2Ga83D34wi1o9J6Wh1mRuqd4mF/x/lgBS7N7AbDhec= @@ -605,15 +604,15 @@ github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5Cc github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7/go.mod h1:q4W45IWZaF22tdD+VEXcAWRA037jwmWEB5VWYORlTpc= -github.com/tidwall/gjson v1.10.2/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= -github.com/tidwall/gjson v1.12.0 h1:61wEp/qfvFnqKH/WCI3M8HuRut+mHT6Mr82QrFmM2SY= -github.com/tidwall/gjson v1.12.0/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= +github.com/tidwall/gjson v1.12.1/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= +github.com/tidwall/gjson v1.14.0 h1:6aeJ0bzojgWLa82gDQHcx3S0Lr/O51I9bJ5nv6JFx5w= +github.com/tidwall/gjson v1.14.0/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA= github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM= github.com/tidwall/pretty v1.2.0 h1:RWIZEg2iJ8/g6fDDYzMpobmaoGh5OLl4AXtGUGPcqCs= github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= -github.com/tidwall/sjson v1.2.3 h1:5+deguEhHSEjmuICXZ21uSSsXotWMA0orU783+Z7Cp8= -github.com/tidwall/sjson v1.2.3/go.mod h1:5WdjKx3AQMvCJ4RG6/2UYT7dLrGvJUV1x4jdTAyGvZs= +github.com/tidwall/sjson v1.2.4 h1:cuiLzLnaMeBhRmEv00Lpk3tkYrcxpmbU81tAY4Dw0tc= +github.com/tidwall/sjson v1.2.4/go.mod h1:098SZ494YoMWPmMO6ct4dcFnqxwj9r/gF0Etp19pSNM= github.com/tinylib/msgp v1.0.2/go.mod h1:+d+yLhGm8mzTaHzB+wgMYrodPfmZrzkirds8fDWklFE= github.com/tklauser/go-sysconf v0.3.5/go.mod h1:MkWzOF4RMCshBAMXuhXJs64Rte09mITnppBXY/rYEFI= github.com/tklauser/numcpus v0.2.2/go.mod h1:x3qojaO3uyYt0i56EW/VUYs7uBvdl2fkfZFu0T9wgjM= @@ -652,6 +651,7 @@ go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqe go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.7.0 h1:ADUqmZGgLDDfbSL9ZmPxKTybcoEYHgpYfELNoN+7hsw= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= +go.uber.org/goleak v1.1.11 h1:wy28qYRKZgnJTxGxvye5/wgWr1EKjmUDGYox5mGlRlI= go.uber.org/goleak v1.1.11/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= go.uber.org/multierr v1.6.0 h1:y6IPFStTAIT5Ytl7/XYmHvzXQ7S3g/IeZW9hyZ5thw4= @@ -1134,6 +1134,7 @@ gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo= diff --git a/pkg/tester/construction.go b/pkg/tester/construction.go index 653b3570..53ff7e82 100644 --- a/pkg/tester/construction.go +++ b/pkg/tester/construction.go @@ -205,6 +205,7 @@ func InitializeConstruction( // Load prefunded accounts var accountBalanceRequests []*utils.AccountBalanceRequest + var acctCoinsReqs []*utils.AccountCoinsRequest for _, prefundedAcc := range config.Construction.PrefundedAccounts { accountBalance := &utils.AccountBalanceRequest{ Account: prefundedAcc.AccountIdentifier, @@ -212,7 +213,15 @@ func InitializeConstruction( Currency: prefundedAcc.Currency, } + acctCoinsReq := &utils.AccountCoinsRequest{ + Account: prefundedAcc.AccountIdentifier, + Network: network, + Currencies: []*types.Currency{prefundedAcc.Currency}, + IncludeMempool: false, + } + accountBalanceRequests = append(accountBalanceRequests, accountBalance) + acctCoinsReqs = append(acctCoinsReqs, acctCoinsReq) } accBalances, err := utils.GetAccountBalances(ctx, onlineFetcher, accountBalanceRequests) @@ -225,11 +234,30 @@ func InitializeConstruction( return nil, fmt.Errorf("%w: unable to set balances", err) } - err = coinStorage.SetCoinsImported(ctx, accBalances) + // ------------------------------------------------------------------------- + // ------------ Get account coins and add them in coins storage ------------ + // ------------------------------------------------------------------------- + + acctCoins, errAccCoins := utils.GetAccountCoins(ctx, onlineFetcher, acctCoinsReqs) + if errAccCoins != nil { + return nil, fmt.Errorf("%w: unable to get account coins", errAccCoins) + } + + // Extract accounts from account coins requests + var accts []*types.AccountIdentifier + for _, req := range acctCoinsReqs { + accts = append(accts, req.Account) + } + + err = coinStorage.SetCoinsImported(ctx, accts, acctCoins) if err != nil { return nil, fmt.Errorf("%w: unable to set coin balances", err) } + // -------------------------------------------------------------------------- + // ---------------------- End of adding account coins ----------------------- + // -------------------------------------------------------------------------- + jobStorage := modules.NewJobStorage(localStore) coordinatorHelper := processor.NewCoordinatorHelper( offlineFetcher, From 3b3f5745c4791408ef8ba34255069451d9064df5 Mon Sep 17 00:00:00 2001 From: shiatcb <93357037+shiatcb@users.noreply.github.com> Date: Mon, 4 Apr 2022 11:35:09 -0400 Subject: [PATCH 299/376] ROSE-231: update rosetta-sdk-go version (#296) --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index b20729ce..b1d0cbb3 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/coinbase/rosetta-cli go 1.16 require ( - github.com/coinbase/rosetta-sdk-go v0.7.6 + github.com/coinbase/rosetta-sdk-go v0.7.7 github.com/fatih/color v1.13.0 github.com/olekukonko/tablewriter v0.0.5 github.com/pkg/errors v0.9.1 diff --git a/go.sum b/go.sum index a6a70135..2af9cd4e 100644 --- a/go.sum +++ b/go.sum @@ -161,8 +161,8 @@ github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWH github.com/cncf/xds/go v0.0.0-20211130200136-a8f946100490/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/coinbase/kryptology v1.8.0 h1:Aoq4gdTsJhSU3lNWsD5BWmFSz2pE0GlmrljaOxepdYY= github.com/coinbase/kryptology v1.8.0/go.mod h1:RYXOAPdzOGUe3qlSFkMGn58i3xUA8hmxYHksuq+8ciI= -github.com/coinbase/rosetta-sdk-go v0.7.6 h1:y3/Lxu9ojIsUKnhNc5UaZF/i0byfuNDWXLYMSKG+IBs= -github.com/coinbase/rosetta-sdk-go v0.7.6/go.mod h1:lt2lJX/HQ2stryuKkukCWMcX3T5fBU62UrdhuwsnVYg= +github.com/coinbase/rosetta-sdk-go v0.7.7 h1:1b4YhFs4hE7sjRlsJ16VBow1cq0NfmKY2v9tT1PKx7I= +github.com/coinbase/rosetta-sdk-go v0.7.7/go.mod h1:lt2lJX/HQ2stryuKkukCWMcX3T5fBU62UrdhuwsnVYg= github.com/consensys/bavard v0.1.8-0.20210406032232-f3452dc9b572/go.mod h1:Bpd0/3mZuaj6Sj+PqrmIquiOKy397AKGThQPaGzNXAQ= github.com/consensys/bavard v0.1.8-0.20210915155054-088da2f7f54a/go.mod h1:9ItSMtA/dXMAiL7BG6bqW2m3NdSEObYWoH223nGHukI= github.com/consensys/gnark-crypto v0.4.1-0.20210426202927-39ac3d4b3f1f/go.mod h1:815PAHg3wvysy0SyIqanF8gZ0Y1wjk/hrDHD/iT88+Q= From 349746114bc6d1f8f077b6a0976e099262bcfc6a Mon Sep 17 00:00:00 2001 From: shiatcb <93357037+shiatcb@users.noreply.github.com> Date: Tue, 5 Apr 2022 14:29:55 -0400 Subject: [PATCH 300/376] ROSE-239: BugFix: Add CoinSupported configuration (#297) --- configuration/types.go | 6 ++++ examples/configuration/default.json | 1 + pkg/tester/construction.go | 45 ++++++++++++++++------------- 3 files changed, 32 insertions(+), 20 deletions(-) diff --git a/configuration/types.go b/configuration/types.go index c0f53c3a..1044143b 100644 --- a/configuration/types.go +++ b/configuration/types.go @@ -425,6 +425,12 @@ type Configuration struct { // if the data or construction check fails ErrorStackTraceDisabled bool `json:"error_stack_trace_disabled"` + // CoinSupported indicates whether your implementation support coins or not. + // If your implementation is based on account-based blockchain (e.g. Ethereum), + // this value must be false. If your implementation is UTXO-based blockchain (e.g. Bitcoin), + // then this value must be true. + CoinSupported bool `json:"coin_supported"` + Construction *ConstructionConfiguration `json:"construction"` Data *DataConfiguration `json:"data"` } diff --git a/examples/configuration/default.json b/examples/configuration/default.json index e119f906..11ee99f3 100644 --- a/examples/configuration/default.json +++ b/examples/configuration/default.json @@ -16,6 +16,7 @@ "compression_disabled": false, "memory_limit_disabled": false, "error_stack_trace_disabled": false, + "coin_supported": false, "construction": null, "data": { "active_reconciliation_concurrency": 16, diff --git a/pkg/tester/construction.go b/pkg/tester/construction.go index 53ff7e82..ac763bbd 100644 --- a/pkg/tester/construction.go +++ b/pkg/tester/construction.go @@ -213,15 +213,18 @@ func InitializeConstruction( Currency: prefundedAcc.Currency, } - acctCoinsReq := &utils.AccountCoinsRequest{ - Account: prefundedAcc.AccountIdentifier, - Network: network, - Currencies: []*types.Currency{prefundedAcc.Currency}, - IncludeMempool: false, - } - accountBalanceRequests = append(accountBalanceRequests, accountBalance) - acctCoinsReqs = append(acctCoinsReqs, acctCoinsReq) + + if config.CoinSupported { + acctCoinsReq := &utils.AccountCoinsRequest{ + Account: prefundedAcc.AccountIdentifier, + Network: network, + Currencies: []*types.Currency{prefundedAcc.Currency}, + IncludeMempool: false, + } + + acctCoinsReqs = append(acctCoinsReqs, acctCoinsReq) + } } accBalances, err := utils.GetAccountBalances(ctx, onlineFetcher, accountBalanceRequests) @@ -238,20 +241,22 @@ func InitializeConstruction( // ------------ Get account coins and add them in coins storage ------------ // ------------------------------------------------------------------------- - acctCoins, errAccCoins := utils.GetAccountCoins(ctx, onlineFetcher, acctCoinsReqs) - if errAccCoins != nil { - return nil, fmt.Errorf("%w: unable to get account coins", errAccCoins) - } + if config.CoinSupported { + acctCoins, errAccCoins := utils.GetAccountCoins(ctx, onlineFetcher, acctCoinsReqs) + if errAccCoins != nil { + return nil, fmt.Errorf("%w: unable to get account coins", errAccCoins) + } - // Extract accounts from account coins requests - var accts []*types.AccountIdentifier - for _, req := range acctCoinsReqs { - accts = append(accts, req.Account) - } + // Extract accounts from account coins requests + var accts []*types.AccountIdentifier + for _, req := range acctCoinsReqs { + accts = append(accts, req.Account) + } - err = coinStorage.SetCoinsImported(ctx, accts, acctCoins) - if err != nil { - return nil, fmt.Errorf("%w: unable to set coin balances", err) + err = coinStorage.SetCoinsImported(ctx, accts, acctCoins) + if err != nil { + return nil, fmt.Errorf("%w: unable to set coin balances", err) + } } // -------------------------------------------------------------------------- From 5d4a2663267b38a5bc4cbca2077658980364bdf7 Mon Sep 17 00:00:00 2001 From: Rizeng Zheng Date: Thu, 7 Apr 2022 13:30:57 -0700 Subject: [PATCH 301/376] Include operations from related transactions to match intent (#283) --- pkg/processor/broadcast_storage_handler.go | 15 ++++++++++++++- pkg/tester/construction.go | 1 + 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/pkg/processor/broadcast_storage_handler.go b/pkg/processor/broadcast_storage_handler.go index 57b881c3..32e617f9 100644 --- a/pkg/processor/broadcast_storage_handler.go +++ b/pkg/processor/broadcast_storage_handler.go @@ -35,6 +35,7 @@ var _ modules.BroadcastStorageHandler = (*BroadcastStorageHandler)(nil) // can be sent to other functions (ex: reconciler). type BroadcastStorageHandler struct { config *configuration.Configuration + blockStorage *modules.BlockStorage counterStorage *modules.CounterStorage coordinator *coordinator.Coordinator parser *parser.Parser @@ -43,12 +44,14 @@ type BroadcastStorageHandler struct { // NewBroadcastStorageHandler returns a new *BroadcastStorageHandler. func NewBroadcastStorageHandler( config *configuration.Configuration, + blockStorage *modules.BlockStorage, counterStorage *modules.CounterStorage, coordinator *coordinator.Coordinator, parser *parser.Parser, ) *BroadcastStorageHandler { return &BroadcastStorageHandler{ config: config, + blockStorage: blockStorage, counterStorage: counterStorage, coordinator: coordinator, parser: parser, @@ -65,7 +68,17 @@ func (h *BroadcastStorageHandler) TransactionConfirmed( transaction *types.Transaction, intent []*types.Operation, ) error { - if err := h.parser.ExpectedOperations(intent, transaction.Operations, false, true); err != nil { + _, _, relatedTransactions, err := h.blockStorage.FindRelatedTransactions(ctx, transaction.TransactionIdentifier, dbTx) + if err != nil { + return fmt.Errorf("%w: could not find related transactions", err) + } + + observed := transaction.Operations + for _, relatedTransaction := range relatedTransactions { + observed = append(observed, relatedTransaction.Operations...) + } + + if err := h.parser.ExpectedOperations(intent, observed, false, true); err != nil { return fmt.Errorf("%w: confirmed transaction did not match intent", err) } diff --git a/pkg/tester/construction.go b/pkg/tester/construction.go index ac763bbd..dd2f19d9 100644 --- a/pkg/tester/construction.go +++ b/pkg/tester/construction.go @@ -294,6 +294,7 @@ func InitializeConstruction( broadcastHandler := processor.NewBroadcastStorageHandler( config, + blockStorage, counterStorage, coordinator, parser, From 165ce8fa1f664f0c196843885410cb6823ea3752 Mon Sep 17 00:00:00 2001 From: Madhur Shrimal Date: Tue, 26 Apr 2022 23:12:41 -0700 Subject: [PATCH 302/376] fix salus issue --- cmd/root.go | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/cmd/root.go b/cmd/root.go index d60d68c3..092e90cb 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -20,6 +20,7 @@ import ( "log" "os" "os/signal" + "path" "runtime" "runtime/pprof" "syscall" @@ -92,7 +93,7 @@ var ( // Bassed on https://golang.org/pkg/runtime/pprof/#hdr-Profiling_a_Go_program func rootPreRun(*cobra.Command, []string) error { if cpuProfile != "" { - f, err := os.Create(cpuProfile) + f, err := os.Create(path.Clean(cpuProfile)) if err != nil { return fmt.Errorf("%w: unable to create CPU profile file", err) } @@ -113,7 +114,7 @@ func rootPreRun(*cobra.Command, []string) error { if blockProfile != "" { runtime.SetBlockProfileRate(1) - f, err := os.Create(blockProfile) + f, err := os.Create(path.Clean(blockProfile)) if err != nil { return fmt.Errorf("%w: unable to create block profile file", err) } @@ -144,7 +145,7 @@ func rootPostRun() { } if memProfile != "" { - f, err := os.Create(memProfile) + f, err := os.Create(path.Clean(memProfile)) if err != nil { log.Printf("error while creating mem-profile file: %v", err) return From 81ef5753cf7ca790f804b23517e7e401233eb8ba Mon Sep 17 00:00:00 2001 From: irisZhangCB <98374544+irisZhangCB@users.noreply.github.com> Date: Wed, 27 Apr 2022 09:15:33 -0700 Subject: [PATCH 303/376] Replace CircleCI with gh actions (#305) * Replace CircleCI with gh actions * fix lint --- .circleci/config.yml | 103 --------------------------------------- .github/workflows/ci.yml | 81 ++++++++++++++++++++++++++++++ Makefile | 3 +- 3 files changed, 83 insertions(+), 104 deletions(-) delete mode 100644 .circleci/config.yml create mode 100644 .github/workflows/ci.yml diff --git a/.circleci/config.yml b/.circleci/config.yml deleted file mode 100644 index 232e169e..00000000 --- a/.circleci/config.yml +++ /dev/null @@ -1,103 +0,0 @@ -# Copyright 2020 Coinbase, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -version: 2.1 -executors: - default: - docker: - - image: cimg/go:1.16 - user: root # go directory is owned by root - working_directory: /go/src/github.com/coinbase/rosetta-cli - environment: - - GO111MODULE: "on" - -fast-checkout: &fast-checkout - attach_workspace: - at: /go - -jobs: - setup: - executor: - name: default - steps: - - checkout - - run: make deps - - persist_to_workspace: - root: /go - paths: - - src - - bin - - pkg/mod/cache - test: - executor: - name: default - steps: - - *fast-checkout - - run: make test - lint: - executor: - name: default - steps: - - *fast-checkout - - run: curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(go env GOPATH)/bin v1.32.2 - - run: make lint - check-license: - executor: - name: default - steps: - - *fast-checkout - - run: make check-license - check-format: - executor: - name: default - steps: - - *fast-checkout - - run: make check-format - coverage: - executor: - name: default - steps: - - *fast-checkout - - run: make test-cover - salus: - machine: true - steps: - - checkout - - run: docker run -t -v $(pwd):/home/repo coinbase/salus - -workflows: - version: 2 - build: - jobs: - - setup - - test: - requires: - - setup - - lint: - requires: - - setup - - check-license: - requires: - - setup - - check-format: - requires: - - setup - - coverage: - requires: - - setup - - salus - -notify: - webhooks: - - url: https://coveralls.io/webhook?repo_token=$COVERALLS_TOKEN diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 00000000..330961a2 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,81 @@ +name: CI + +on: + push: + branches: + - master + pull_request: + +env: + go_version: 1.16 + GO111MODULE: on + COVERALLS_TOKEN: ${{ secrets.COVERALLS_TOKEN }} + +jobs: + Build: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-go@v3 + with: + go-version: ${{ env.go_version }} + - run: make build + Test: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-go@v3 + with: + go-version: ${{ env.go_version }} + - run: make test + Lint: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-go@v3 + with: + go-version: ${{ env.go_version }} + - uses: golangci/golangci-lint-action@v3 + with: + version: latest + args: --timeout 3m + + Check-License: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + with: + version: latest + - uses: actions/setup-go@v3 + with: + go-version: ${{ env.go_version }} + - run: make check-license + + Check-format: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + with: + version: latest + - run: make check-format + + Coverage: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + with: + version: latest + - uses: actions/setup-go@v3 + with: + go-version: ${{ env.go_version }} + - run: make test-cover COVERALLS_TOKEN="$COVERALLS_TOKEN" + + Salus: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + with: + version: latest + - run: make salus + + \ No newline at end of file diff --git a/Makefile b/Makefile index 5fdf9434..c2487590 100644 --- a/Makefile +++ b/Makefile @@ -9,7 +9,8 @@ # the commands directly. ADDLICENSE_INSTALL=go install github.com/google/addlicense@latest ADDLICENSE_CMD=addlicense -ADDLICENCE_SCRIPT=${ADDLICENSE_CMD} -c "Coinbase, Inc." -l "apache" -v +ADDLICENSE_IGNORE=-ignore ".github/**/*" -ignore ".idea/**/*" +ADDLICENCE_SCRIPT=${ADDLICENSE_CMD} -c "Coinbase, Inc." -l "apache" -v ${ADDLICENSE_IGNORE} GOLINES_INSTALL=go install github.com/segmentio/golines@latest GOLINES_CMD=golines GOVERALLS_INSTALL=go install github.com/mattn/goveralls@latest From 15c2f149764d0a7c9b6511a0e89ddbea40493766 Mon Sep 17 00:00:00 2001 From: Madhur Shrimal Date: Thu, 21 Apr 2022 16:24:52 -0700 Subject: [PATCH 304/376] update rsdk --- go.mod | 7 +++++-- go.sum | 3 +++ 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/go.mod b/go.mod index b1d0cbb3..26966264 100644 --- a/go.mod +++ b/go.mod @@ -3,12 +3,15 @@ module github.com/coinbase/rosetta-cli go 1.16 require ( - github.com/coinbase/rosetta-sdk-go v0.7.7 + github.com/Azure/azure-pipeline-go v0.2.2 // indirect + github.com/Azure/azure-storage-blob-go v0.7.0 // indirect + github.com/Azure/go-autorest/autorest/adal v0.8.0 // indirect + github.com/coinbase/rosetta-sdk-go v0.7.8 github.com/fatih/color v1.13.0 github.com/olekukonko/tablewriter v0.0.5 github.com/pkg/errors v0.9.1 github.com/spf13/cobra v1.3.0 - github.com/stretchr/testify v1.7.0 + github.com/stretchr/testify v1.7.1 go.uber.org/zap v1.21.0 golang.org/x/sync v0.0.0-20210220032951-036812b2e83c ) diff --git a/go.sum b/go.sum index 2af9cd4e..dd60ea4e 100644 --- a/go.sum +++ b/go.sum @@ -163,6 +163,8 @@ github.com/coinbase/kryptology v1.8.0 h1:Aoq4gdTsJhSU3lNWsD5BWmFSz2pE0GlmrljaOxe github.com/coinbase/kryptology v1.8.0/go.mod h1:RYXOAPdzOGUe3qlSFkMGn58i3xUA8hmxYHksuq+8ciI= github.com/coinbase/rosetta-sdk-go v0.7.7 h1:1b4YhFs4hE7sjRlsJ16VBow1cq0NfmKY2v9tT1PKx7I= github.com/coinbase/rosetta-sdk-go v0.7.7/go.mod h1:lt2lJX/HQ2stryuKkukCWMcX3T5fBU62UrdhuwsnVYg= +github.com/coinbase/rosetta-sdk-go v0.7.8 h1:op/O3/ZngTfcrZnp3p/TziRfKGdo7AUZGUmBu6+8qCc= +github.com/coinbase/rosetta-sdk-go v0.7.8/go.mod h1:vB6hZ0ZnZmln3ThA4x0mZvOAPDJ5BhfgnjH76hxoy10= github.com/consensys/bavard v0.1.8-0.20210406032232-f3452dc9b572/go.mod h1:Bpd0/3mZuaj6Sj+PqrmIquiOKy397AKGThQPaGzNXAQ= github.com/consensys/bavard v0.1.8-0.20210915155054-088da2f7f54a/go.mod h1:9ItSMtA/dXMAiL7BG6bqW2m3NdSEObYWoH223nGHukI= github.com/consensys/gnark-crypto v0.4.1-0.20210426202927-39ac3d4b3f1f/go.mod h1:815PAHg3wvysy0SyIqanF8gZ0Y1wjk/hrDHD/iT88+Q= @@ -602,6 +604,7 @@ github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5 github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7/go.mod h1:q4W45IWZaF22tdD+VEXcAWRA037jwmWEB5VWYORlTpc= github.com/tidwall/gjson v1.12.1/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= From bbc39913b99feafb37633f30a988e4150fb9d992 Mon Sep 17 00:00:00 2001 From: Madhur Shrimal Date: Fri, 29 Apr 2022 10:52:41 -0700 Subject: [PATCH 305/376] update to latest rosetta-sdk-go --- go.mod | 5 +---- go.sum | 50 ++++++++++++++++++++++++-------------------------- 2 files changed, 25 insertions(+), 30 deletions(-) diff --git a/go.mod b/go.mod index 26966264..f572db24 100644 --- a/go.mod +++ b/go.mod @@ -3,10 +3,7 @@ module github.com/coinbase/rosetta-cli go 1.16 require ( - github.com/Azure/azure-pipeline-go v0.2.2 // indirect - github.com/Azure/azure-storage-blob-go v0.7.0 // indirect - github.com/Azure/go-autorest/autorest/adal v0.8.0 // indirect - github.com/coinbase/rosetta-sdk-go v0.7.8 + github.com/coinbase/rosetta-sdk-go v0.7.9 github.com/fatih/color v1.13.0 github.com/olekukonko/tablewriter v0.0.5 github.com/pkg/errors v0.9.1 diff --git a/go.sum b/go.sum index dd60ea4e..cebc8547 100644 --- a/go.sum +++ b/go.sum @@ -54,19 +54,9 @@ filippo.io/edwards25519 v1.0.0-rc.1 h1:m0VOOB23frXZvAOK44usCgLWvtsxIoMCTBGJZlpmG filippo.io/edwards25519 v1.0.0-rc.1/go.mod h1:N1IkdkCkiLB6tki+MYJoSx2JTY9NUlxZE7eHn5EwJns= git.sr.ht/~sircmpwn/getopt v0.0.0-20191230200459-23622cc906b3/go.mod h1:wMEGFFFNuPos7vHmWXfszqImLppbc0wEhh6JBfJIUgw= git.sr.ht/~sircmpwn/go-bare v0.0.0-20210406120253-ab86bc2846d9/go.mod h1:BVJwbDfVjCjoFiKrhkei6NdGcZYpkDkdyCdg1ukytRA= -github.com/Azure/azure-pipeline-go v0.2.1/go.mod h1:UGSo8XybXnIGZ3epmeBw7Jdz+HiUVpqIlpz/HKHylF4= -github.com/Azure/azure-pipeline-go v0.2.2/go.mod h1:4rQ/NZncSvGqNkkOsNpOU1tgoNuIlp9AfUH5G1tvCHc= -github.com/Azure/azure-storage-blob-go v0.7.0/go.mod h1:f9YQKtsG1nMisotuTPpO0tjNuEjKRYAcJU8/ydDI++4= -github.com/Azure/go-autorest/autorest v0.9.0/go.mod h1:xyHB1BMZT0cuDHU7I0+g046+BFDTQ8rEZB0s4Yfa6bI= -github.com/Azure/go-autorest/autorest/adal v0.5.0/go.mod h1:8Z9fGy2MpX0PvDjB1pEgQTmVqjGhiHBW7RJJEciWzS0= -github.com/Azure/go-autorest/autorest/adal v0.8.0/go.mod h1:Z6vX6WXXuyieHAXwMj0S6HY6e6wcHn37qQMBQlvY3lc= -github.com/Azure/go-autorest/autorest/date v0.1.0/go.mod h1:plvfp3oPSKwf2DNjlBjWF/7vwR+cUD/ELuzDCXwHUVA= -github.com/Azure/go-autorest/autorest/date v0.2.0/go.mod h1:vcORJHLJEh643/Ioh9+vPmf1Ij9AEBM5FuBIXLmIy0g= -github.com/Azure/go-autorest/autorest/mocks v0.1.0/go.mod h1:OTyCOPRA2IgIlWxVYxBee2F5Gr4kF2zd2J5cFRaIDN0= -github.com/Azure/go-autorest/autorest/mocks v0.2.0/go.mod h1:OTyCOPRA2IgIlWxVYxBee2F5Gr4kF2zd2J5cFRaIDN0= -github.com/Azure/go-autorest/autorest/mocks v0.3.0/go.mod h1:a8FDP3DYzQ4RYfVAxAN3SVSiiO77gL2j2ronKKP0syM= -github.com/Azure/go-autorest/logger v0.1.0/go.mod h1:oExouG+K6PryycPJfVSxi/koC6LSNgds39diKLz7Vrc= -github.com/Azure/go-autorest/tracing v0.5.0/go.mod h1:r/s2XiOKccPW3HrqB+W0TQzfbtp2fGCgRFtBroKn4Dk= +github.com/Azure/azure-sdk-for-go/sdk/azcore v0.21.1/go.mod h1:fBF9PQNqB8scdgpZ3ufzaLntG0AG7C1WjPMsiFOmfHM= +github.com/Azure/azure-sdk-for-go/sdk/internal v0.8.3/go.mod h1:KLF4gFr6DcKFZwSuH8w8yEK6DpFl3LP5rhdvAb7Yz5I= +github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v0.3.0/go.mod h1:tPaiy8S5bQ+S5sOiDlINkp7+Ef339+Nz5L5XO+cnOHo= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/DATA-DOG/go-sqlmock v1.3.3/go.mod h1:f/Ixk793poVmq4qj/V1dPUg2JEAKC73Q5eFN3EC/SaM= @@ -115,8 +105,12 @@ github.com/boltdb/bolt v1.3.1/go.mod h1:clJnj/oiGkjum5o1McbSZDSLxVThjynRyGBgiAx2 github.com/btcsuite/btcd v0.0.0-20190315201642-aa6e0f35703c/go.mod h1:DrZx5ec/dmnfpw9KyYoQyYo7d0KEvTkk/5M/vbZjAr8= github.com/btcsuite/btcd v0.20.1-beta/go.mod h1:wVuoA8VJLEcwgqHBwHmzLRazpKxTv13Px/pDuV7OomQ= github.com/btcsuite/btcd v0.21.0-beta.0.20201114000516-e9c7a5ac6401/go.mod h1:Sv4JPQ3/M+teHz9Bo5jBpkNcP0x6r7rdihlNL/7tTAs= -github.com/btcsuite/btcd v0.22.0-beta h1:LTDpDKUM5EeOFBPM8IXpinEcmZ6FWfNZbE3lfrfdnWo= -github.com/btcsuite/btcd v0.22.0-beta/go.mod h1:9n5ntfhhHQBIhUvlhDvD3Qg6fRUj4jkN0VB8L8svzOA= +github.com/btcsuite/btcd v0.22.1 h1:CnwP9LM/M9xuRrGSCGeMVs9iv09uMqwsVX7EeIpgV2c= +github.com/btcsuite/btcd v0.22.1/go.mod h1:wqgTSL29+50LRkmOVknEdmt8ZojIzhuWvgu/iptuN7Y= +github.com/btcsuite/btcd/btcec/v2 v2.1.2/go.mod h1:ctjw4H1kknNJmRN4iP1R7bTQ+v3GJkZBd6mui8ZsAZE= +github.com/btcsuite/btcd/chaincfg/chainhash v1.0.0/go.mod h1:7SFka0XMvUgj3hfZtydOrQY2mwhPclbT2snogU7SQQc= +github.com/btcsuite/btcd/chaincfg/chainhash v1.0.1 h1:q0rUy8C/TYNBQS1+CGKw68tLOFYSNEs0TFnxxnS9+4U= +github.com/btcsuite/btcd/chaincfg/chainhash v1.0.1/go.mod h1:7SFka0XMvUgj3hfZtydOrQY2mwhPclbT2snogU7SQQc= github.com/btcsuite/btclog v0.0.0-20170628155309-84c8d2346e9f/go.mod h1:TdznJufoqS23FtqVCzL0ZqgP5MqXbb4fg/WgDys70nA= github.com/btcsuite/btcutil v0.0.0-20190207003914-4c204d697803/go.mod h1:+5NJ2+qvTyV9exUAL/rxXi3DcLg2Ts+ymUAY5y4NvMg= github.com/btcsuite/btcutil v0.0.0-20190425235716-9e5f4b9a998d/go.mod h1:+5NJ2+qvTyV9exUAL/rxXi3DcLg2Ts+ymUAY5y4NvMg= @@ -161,10 +155,8 @@ github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWH github.com/cncf/xds/go v0.0.0-20211130200136-a8f946100490/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/coinbase/kryptology v1.8.0 h1:Aoq4gdTsJhSU3lNWsD5BWmFSz2pE0GlmrljaOxepdYY= github.com/coinbase/kryptology v1.8.0/go.mod h1:RYXOAPdzOGUe3qlSFkMGn58i3xUA8hmxYHksuq+8ciI= -github.com/coinbase/rosetta-sdk-go v0.7.7 h1:1b4YhFs4hE7sjRlsJ16VBow1cq0NfmKY2v9tT1PKx7I= -github.com/coinbase/rosetta-sdk-go v0.7.7/go.mod h1:lt2lJX/HQ2stryuKkukCWMcX3T5fBU62UrdhuwsnVYg= -github.com/coinbase/rosetta-sdk-go v0.7.8 h1:op/O3/ZngTfcrZnp3p/TziRfKGdo7AUZGUmBu6+8qCc= -github.com/coinbase/rosetta-sdk-go v0.7.8/go.mod h1:vB6hZ0ZnZmln3ThA4x0mZvOAPDJ5BhfgnjH76hxoy10= +github.com/coinbase/rosetta-sdk-go v0.7.9 h1:lqllBjMnazTjIqYrOGv8h8jxjg9+hJazIGZr9ZvoCcA= +github.com/coinbase/rosetta-sdk-go v0.7.9/go.mod h1:0/knutI7XGVqXmmH4OQD8OckFrbQ8yMsUZTG7FXCR2M= github.com/consensys/bavard v0.1.8-0.20210406032232-f3452dc9b572/go.mod h1:Bpd0/3mZuaj6Sj+PqrmIquiOKy397AKGThQPaGzNXAQ= github.com/consensys/bavard v0.1.8-0.20210915155054-088da2f7f54a/go.mod h1:9ItSMtA/dXMAiL7BG6bqW2m3NdSEObYWoH223nGHukI= github.com/consensys/gnark-crypto v0.4.1-0.20210426202927-39ac3d4b3f1f/go.mod h1:815PAHg3wvysy0SyIqanF8gZ0Y1wjk/hrDHD/iT88+Q= @@ -186,6 +178,8 @@ github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/deckarep/golang-set v1.8.0/go.mod h1:5nI87KwE7wgsBU1F4GKAw2Qod7p5kyS383rP6+o6qqo= +github.com/decred/dcrd/crypto/blake256 v1.0.0/go.mod h1:sQl2p6Y26YV+ZOcSTP6thNdn47hh8kt6rqSlvmrXFAc= +github.com/decred/dcrd/dcrec/secp256k1/v4 v4.0.1/go.mod h1:hyedUtir6IdtD/7lIxGeCxkaw7y45JueMRL4DIyJDKs= github.com/decred/dcrd/lru v1.0.0/go.mod h1:mxKOwFd7lFjN2GZYsiz/ecgqR6kkYAl+0pz0tEMk218= github.com/deepmap/oapi-codegen v1.6.0/go.mod h1:ryDa9AgbELGeB+YEXE1dR53yAjHwFvE9iAUlWl9Al3M= github.com/deepmap/oapi-codegen v1.8.2/go.mod h1:YLgSKSDv/bZQB7N4ws6luhozi3cEdRktEqrX88CvjIw= @@ -201,6 +195,8 @@ github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 h1:fAjc9m62+UWV/WA github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= github.com/dlclark/regexp2 v1.4.1-0.20201116162257-a2a8dda75c91/go.mod h1:2pZnwuY/m+8K6iRw6wQdMtk+rH5tNGR1i55kozfMjCc= +github.com/dnaeon/go-vcr v1.1.0/go.mod h1:M7tiix8f0r6mKKJ3Yq/kqU1OYf3MnfmBWVbPx/yU9ko= +github.com/dnaeon/go-vcr v1.2.0/go.mod h1:R4UdLID7HZT3taECzJs4YgbbH6PIGXB6W/sc5OLb6RQ= github.com/docker/docker v1.4.2-0.20180625184442-8e610b2b55bf/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= github.com/dop251/goja v0.0.0-20211011172007-d99e4b8cbf48/go.mod h1:R9ET47fwRVRPZnOGvHxxhuZcbrMCuiqOz3Rlrh4KSnk= github.com/dop251/goja_nodejs v0.0.0-20210225215109-d91c329300e7/go.mod h1:hn7BA7c8pLvoGndExHudxTDKZ84Pyvv+90pbBjbTz0Y= @@ -219,8 +215,8 @@ github.com/envoyproxy/go-control-plane v0.9.10-0.20210907150352-cf90f659a021/go. github.com/envoyproxy/go-control-plane v0.10.1/go.mod h1:AY7fTTXNdv/aJ2O5jwpxAPOWUZ7hQAEvzN5Pf27BkQQ= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/envoyproxy/protoc-gen-validate v0.6.2/go.mod h1:2t7qjJNvHPx8IjnBOzl9E9/baC+qXE/TeeyBRzgJDws= -github.com/ethereum/go-ethereum v1.10.16 h1:3oPrumn0bCW/idjcxMn5YYVCdK7VzJYIvwGZUGLEaoc= -github.com/ethereum/go-ethereum v1.10.16/go.mod h1:Anj6cxczl+AHy63o4X9O8yWNHuN5wMpfb8MAnHkWn7Y= +github.com/ethereum/go-ethereum v1.10.17 h1:XEcumY+qSr1cZQaWsQs5Kck3FHB0V2RiMHPdTBJ+oT8= +github.com/ethereum/go-ethereum v1.10.17/go.mod h1:Lt5WzjM07XlXc95YzrhosmR4J9Ahd6X2wyEV2SvGhk0= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/fatih/color v1.9.0/go.mod h1:eQcE1qtQxscV5RaZvpXrrb8Drkc3/DdQ+uUYCNjL+zU= github.com/fatih/color v1.13.0 h1:8LOYc1KYPPmyKMuN8QV2DNRWNbLo6LZ0iLs8+mlH53w= @@ -255,6 +251,7 @@ github.com/gofrs/uuid v3.3.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRx github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= +github.com/golang-jwt/jwt/v4 v4.3.0/go.mod h1:/xlHOz8bRuivTWchD4jCa+NbatV+wEUSzwAxVc6locg= github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0/go.mod h1:E/TSTwGwJL78qG/PmXZO1EjYhfJinVAhrmmHX6Z8B9k= github.com/golang/geo v0.0.0-20190916061304-5b978397cfec/go.mod h1:QZ0nwyI2jOfgRAoBvP+ab5aRr7c9x7lhGEJrKvBwjWI= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= @@ -333,7 +330,7 @@ github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1/go.mod h1:kpwsk12EmLe github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/google/uuid v1.1.5/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.2.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= github.com/googleapis/gax-go/v2 v2.1.0/go.mod h1:Q3nei7sK6ybPYH7twZdmQpAd1MKb7pfu6SK+H1/DsU0= @@ -380,7 +377,7 @@ github.com/hashicorp/serf v0.9.6/go.mod h1:TXZNMjZQijwlDvp+r0b63xZ45H7JmCmgg4gpT github.com/holiman/bloomfilter/v2 v2.0.3/go.mod h1:zpoh+gs7qcpqrHr3dB55AMiJwo0iURXE7ZOP9L9hSkA= github.com/holiman/uint256 v1.2.0/go.mod h1:y4ga/t+u+Xwd7CpDgZESaRcWy0I7XMlTMA25ApIH5Jw= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= -github.com/huin/goupnp v1.0.2/go.mod h1:0dxJBVBHqTMjIUMkESDTNgOOx/Mw5wYIfyFmdzSamkM= +github.com/huin/goupnp v1.0.3-0.20220313090229-ca81a64b4204/go.mod h1:ZxNlw5WqJj6wSsRK5+YfflQGXYfccj5VgQsMNixHM7Y= github.com/huin/goutil v0.0.0-20170803182201-1ca381bf3150/go.mod h1:PpLOETDnJ0o3iZrZfqZzyLl6l7F3c6L1oWn7OICBi6o= github.com/iancoleman/strcase v0.2.0/go.mod h1:iwCmte+B7n89clKwxIoIXy/HfoL7AsD47ZCWhYzw7ho= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= @@ -461,8 +458,6 @@ github.com/mattn/go-colorable v0.1.8/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope github.com/mattn/go-colorable v0.1.9/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= github.com/mattn/go-colorable v0.1.12 h1:jF+Du6AlPIjs2BiUiQlKOX0rt3SujHxPnksPKZbaA40= github.com/mattn/go-colorable v0.1.12/go.mod h1:u5H1YNBxpqRaxsYJYSkiCWKzEfiAb1Gb520KVy5xxl4= -github.com/mattn/go-ieproxy v0.0.0-20190610004146-91bb50d98149/go.mod h1:31jz6HNzdxOmlERGGEc4v/dMssOfmp2p5bT/okiKFFc= -github.com/mattn/go-ieproxy v0.0.0-20190702010315-6dee0af9227d/go.mod h1:31jz6HNzdxOmlERGGEc4v/dMssOfmp2p5bT/okiKFFc= github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= @@ -496,6 +491,7 @@ github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJ github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/modocache/gover v0.0.0-20171022184752-b58185e213c5/go.mod h1:caMODM3PzxT8aQXRPkAt8xlV/e7d7w8GM5g0fa5F0D8= github.com/mschoch/smat v0.0.0-20160514031455-90eadee771ae/go.mod h1:qAyveg+e4CE+eKJXWVjKXM4ck2QobLqTDytGJbLLhJg= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/naoina/go-stringutil v0.1.0/go.mod h1:XJ2SJL9jCtBh+P9q5btrd/Ylo8XwT/h1USek5+NqSA0= @@ -602,8 +598,8 @@ github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UV github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.1 h1:5TQK59W5E3v0r2duFAb7P95B6hEeOyEnHRa8MjYSMTY= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7/go.mod h1:q4W45IWZaF22tdD+VEXcAWRA037jwmWEB5VWYORlTpc= @@ -758,6 +754,7 @@ golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81R golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20200813134508-3edf25e44fcc/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20201010224723-4f7140c49acb/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201031054903-ff519b6c9102/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= @@ -769,6 +766,7 @@ golang.org/x/net v0.0.0-20210316092652-d523dce5a7f4/go.mod h1:RBQZq4jEuRlivfhVLd golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= golang.org/x/net v0.0.0-20210410081132-afb366fc7cd1/go.mod h1:9tjilg8BloeKEkVJvy7fQ90B1CfIiPueXVOjqfkSzI8= golang.org/x/net v0.0.0-20210503060351-7fd8e65b6420/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20210610132358-84b48f89b13b/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210813160813-60bc85c4be6d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f h1:OfiFi4JbukWwe3lzw+xunroH1mnC1e2Gy5cxNJApiSY= From 5986e92cb75bd7439e37363a30d97670c8233798 Mon Sep 17 00:00:00 2001 From: irisZhangCB <98374544+irisZhangCB@users.noreply.github.com> Date: Fri, 29 Apr 2022 10:59:43 -0700 Subject: [PATCH 306/376] error categorization (#303) rename error pkg --- cmd/check_construction.go | 4 +-- cmd/utils_asserter_configuration.go | 2 +- cmd/validate_asserter_config.go | 8 ++--- cmd/view_block.go | 4 +-- cmd/view_networks.go | 4 +-- configuration/configuration.go | 8 +++-- pkg/errors/errors.go | 45 +++++++++++++++++++++++++++++ pkg/tester/construction.go | 11 +++---- 8 files changed, 67 insertions(+), 19 deletions(-) create mode 100644 pkg/errors/errors.go diff --git a/cmd/check_construction.go b/cmd/check_construction.go index 2a04aa79..617a4a0d 100644 --- a/cmd/check_construction.go +++ b/cmd/check_construction.go @@ -16,8 +16,8 @@ package cmd import ( "context" - "errors" "fmt" + "github.com/coinbase/rosetta-cli/pkg/errors" "time" "github.com/coinbase/rosetta-cli/pkg/results" @@ -59,7 +59,7 @@ func runCheckConstructionCmd(_ *cobra.Command, _ []string) error { Config, nil, nil, - errors.New("construction configuration is missing"), + errors.ErrConstructionConfigMissing, ) } diff --git a/cmd/utils_asserter_configuration.go b/cmd/utils_asserter_configuration.go index 3f4b4e83..39afbdbe 100644 --- a/cmd/utils_asserter_configuration.go +++ b/cmd/utils_asserter_configuration.go @@ -81,4 +81,4 @@ func sortArrayFieldsOnConfiguration(configuration *asserter.Configuration) { sort.Slice(configuration.AllowedErrors, func(i, j int) bool { return configuration.AllowedErrors[i].Code < configuration.AllowedErrors[j].Code }) -} \ No newline at end of file +} diff --git a/cmd/validate_asserter_config.go b/cmd/validate_asserter_config.go index 068deba7..ba5afe38 100644 --- a/cmd/validate_asserter_config.go +++ b/cmd/validate_asserter_config.go @@ -16,8 +16,8 @@ package cmd import ( "context" - "errors" "fmt" + "github.com/coinbase/rosetta-cli/pkg/errors" "github.com/coinbase/rosetta-sdk-go/asserter" "github.com/coinbase/rosetta-sdk-go/fetcher" "github.com/coinbase/rosetta-sdk-go/types" @@ -52,10 +52,10 @@ func validateNetworkAndAsserterAllowMatch( networkAllow *types.Allow, asserterConfiguration *asserter.Configuration, ) error { if networkAllow == nil { - return errors.New("/network/options object's Allow is nil") + return fmt.Errorf("%w: /network/options object's Allow is nil", errors.ErrAsserterConfigError) } if asserterConfiguration == nil { - return errors.New("asserter-configuration-file object is nil") + return fmt.Errorf("%w: asserter-configuration-file object is nil", errors.ErrAsserterConfigError) } if err := verifyTimestampStartIndex( @@ -111,7 +111,7 @@ func verifyOperationTypes(networkOt, asserterOt []string) error { if networkOperationType != asserterOperationType { return fmt.Errorf( "/network/options / asserter-configuration-file operation type mismatch %+v "+ - "%+v\nnetwork operation types: %+v\nasserter operation types: %+v", + "%+v\nnetwork operation types: %+v\nasserter operation types: %+v", networkOperationType, asserterOperationType, networkOt, asserterOt, ) } diff --git a/cmd/view_block.go b/cmd/view_block.go index 597d34b5..be2de146 100644 --- a/cmd/view_block.go +++ b/cmd/view_block.go @@ -15,8 +15,8 @@ package cmd import ( - "errors" "fmt" + "github.com/coinbase/rosetta-cli/pkg/errors" "strconv" "time" @@ -125,7 +125,7 @@ func runViewBlockCmd(_ *cobra.Command, args []string) error { } // It's valid for a block to be omitted without triggering an error if block == nil { - return errors.New("block not found, it might be omitted") + return fmt.Errorf("%w: block not found, it might be omitted", errors.ErrBlockNotFound) } fmt.Printf("\n") diff --git a/cmd/view_networks.go b/cmd/view_networks.go index eda06a50..c2d0a478 100644 --- a/cmd/view_networks.go +++ b/cmd/view_networks.go @@ -15,8 +15,8 @@ package cmd import ( - "errors" "fmt" + "github.com/coinbase/rosetta-cli/pkg/errors" "log" "time" @@ -63,7 +63,7 @@ func runViewNetworksCmd(cmd *cobra.Command, args []string) error { } if len(networkList.NetworkIdentifiers) == 0 { - return errors.New("no networks available") + return errors.ErrNoAvailableNetwork } for _, network := range networkList.NetworkIdentifiers { diff --git a/configuration/configuration.go b/configuration/configuration.go index 1dc1a2f9..c92c2cd2 100644 --- a/configuration/configuration.go +++ b/configuration/configuration.go @@ -24,6 +24,7 @@ import ( "runtime" "strings" + customerrors "github.com/coinbase/rosetta-cli/pkg/errors" "github.com/coinbase/rosetta-sdk-go/asserter" "github.com/coinbase/rosetta-sdk-go/constructor/dsl" "github.com/coinbase/rosetta-sdk-go/constructor/job" @@ -180,11 +181,11 @@ func assertConstructionConfiguration(ctx context.Context, config *ConstructionCo } if len(config.Workflows) > 0 && len(config.ConstructorDSLFile) > 0 { - return errors.New("cannot populate both workflows and DSL file path") + return fmt.Errorf("%w: cannot populate both workflows and DSL file path", customerrors.ErrParseFileFailed) } if len(config.Workflows) == 0 && len(config.ConstructorDSLFile) == 0 { - return errors.New("both workflows and DSL file path are empty") + return fmt.Errorf("%w: both workflows and DSL file path are empty", customerrors.ErrParseFileFailed) } // Compile ConstructorDSLFile and save to Workflows @@ -203,7 +204,8 @@ func assertConstructionConfiguration(ctx context.Context, config *ConstructionCo if workflow.Name == string(job.CreateAccount) || workflow.Name == string(job.RequestFunds) { if workflow.Concurrency != job.ReservedWorkflowConcurrency { return fmt.Errorf( - "reserved workflow %s must have concurrency %d", + "%w: reserved workflow %s must have concurrency %d", + customerrors.ErrParseWorkflowFailed, workflow.Name, job.ReservedWorkflowConcurrency, ) diff --git a/pkg/errors/errors.go b/pkg/errors/errors.go new file mode 100644 index 00000000..cac440e4 --- /dev/null +++ b/pkg/errors/errors.go @@ -0,0 +1,45 @@ +// Copyright 2020 Coinbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package errors + +import ( + "errors" +) + +// Configuration Errors + +var ( + // Construction Configuration Errors + + ErrParseFileFailed = errors.New("unable to parse config files") + ErrBalanceTrackingDisabled = errors.New("balance tracking disabled") + ErrReconciliationConfig = errors.New("invalid reconciliation error") + ErrCompileDSLFileFailed = errors.New("unable to compile DSL file") + ErrParseWorkflowFailed = errors.New("unable to parse workflow") + ErrConstructionConfigMissing = errors.New("construction configuration is missing") + + // TODO: Data Configuration Errors + + // Construction check errors + + ErrConstructionCheckHalt = errors.New("construction check halted") + + ErrAsserterConfigError = errors.New("asserter configuration validation failed") + + // Bad Command Errors + + ErrBlockNotFound = errors.New("block not found") + ErrNoAvailableNetwork = errors.New("no networks available") +) diff --git a/pkg/tester/construction.go b/pkg/tester/construction.go index dd2f19d9..537890ca 100644 --- a/pkg/tester/construction.go +++ b/pkg/tester/construction.go @@ -19,6 +19,7 @@ import ( "encoding/json" "errors" "fmt" + customErrs "github.com/coinbase/rosetta-cli/pkg/errors" "log" "net/http" "time" @@ -83,7 +84,7 @@ func InitializeConstruction( ) (*ConstructionTester, error) { dataPath, err := utils.CreateCommandPath(config.DataDirectory, constructionCmdName, network) if err != nil { - log.Fatalf("%s: cannot create command path", err.Error()) + return nil, fmt.Errorf("%s: cannot create command path", err.Error()) } opts := []database.BadgerOption{} @@ -99,17 +100,17 @@ func InitializeConstruction( localStore, err := database.NewBadgerDatabase(ctx, dataPath, opts...) if err != nil { - log.Fatalf("%s: unable to initialize database", err.Error()) + return nil, fmt.Errorf("%s: unable to initialize database", err.Error()) } networkOptions, fetchErr := onlineFetcher.NetworkOptionsRetry(ctx, network, nil) if err != nil { - log.Fatalf("%s: unable to get network options", fetchErr.Err.Error()) + return nil, fmt.Errorf("%s: unable to get network options", fetchErr.Err.Error()) } if len(networkOptions.Allow.BalanceExemptions) > 0 && config.Construction.InitialBalanceFetchDisabled { - log.Fatal("found balance exemptions but initial balance fetch disabled") + return nil, fmt.Errorf("found balance exemptions but initial balance fetch disabled") } counterStorage := modules.NewCounterStorage(localStore) @@ -578,7 +579,7 @@ func (t *ConstructionTester) HandleErr( t.config, t.counterStorage, t.jobStorage, - errors.New("check halted"), + fmt.Errorf("%w: %v", customErrs.ErrConstructionCheckHalt, err.Error()), ) } From 8d2cd993136a0f795aeea70de62c015e109259c7 Mon Sep 17 00:00:00 2001 From: shiatcb <93357037+shiatcb@users.noreply.github.com> Date: Mon, 2 May 2022 13:45:51 -0400 Subject: [PATCH 307/376] ROSE-251: rosetta-cli version should return latest release (#298) --- README.md | 3 +++ cmd/root.go | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index b40f2502..7e50c0be 100644 --- a/README.md +++ b/README.md @@ -120,4 +120,7 @@ You can also find community implementations for a variety of blockchains in the ## License This project is available open source under the terms of the [Apache 2.0 License](https://opensource.org/licenses/Apache-2.0). +## Release +When we release a new rosetta-cli version, please update the [version number](https://github.com/coinbase/rosetta-cli/blob/master/cmd/root.go#L297) so that `rosetta-cli version` command can print the correct value. + © 2022 Coinbase \ No newline at end of file diff --git a/cmd/root.go b/cmd/root.go index 092e90cb..649bd2f6 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -295,6 +295,6 @@ var versionCmd = &cobra.Command{ Use: "version", Short: "Print rosetta-cli version", Run: func(cmd *cobra.Command, args []string) { - fmt.Println("v0.7.3") + fmt.Println("v0.7.6") }, } From ab36fd35e835fb54503f5cf718d49cd3b6d50ae4 Mon Sep 17 00:00:00 2001 From: iris-zhang Date: Tue, 10 May 2022 10:20:46 -0700 Subject: [PATCH 308/376] Add dockerfile --- Dockerfile | 49 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 49 insertions(+) create mode 100644 Dockerfile diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 00000000..7813543f --- /dev/null +++ b/Dockerfile @@ -0,0 +1,49 @@ +# Copyright 2020 Coinbase, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Compile golang +FROM ubuntu:20.04 as cli + +RUN apt-get update && apt-get install -y curl make gcc g++ git +ENV GOLANG_VERSION 1.17.9 +ENV GOLANG_DOWNLOAD_URL https://golang.org/dl/go$GOLANG_VERSION.linux-amd64.tar.gz +ENV GOLANG_DOWNLOAD_SHA256 9dacf782028fdfc79120576c872dee488b81257b1c48e9032d122cfdb379cca6 + +RUN curl -fsSL "$GOLANG_DOWNLOAD_URL" -o golang.tar.gz \ + && echo "$GOLANG_DOWNLOAD_SHA256 golang.tar.gz" | sha256sum -c - \ + && tar -C /usr/local -xzf golang.tar.gz \ + && rm golang.tar.gz + +ENV GOPATH /go +ENV PATH $GOPATH/bin:/usr/local/go/bin:$PATH +RUN mkdir -p "$GOPATH/src" "$GOPATH/bin" && chmod -R 777 "$GOPATH" + +WORKDIR /go/src + +ARG VERSION=v0.7.6 +RUN git clone https://github.com/coinbase/rosetta-cli.git && \ + cd rosetta-cli && \ + git fetch --all --tags && \ + git checkout $VERSION && \ + make install + +FROM ubuntu:20.04 + +RUN apt-get update -y && apt-get install -y \ + curl + +# Copy all the binaries +COPY --from=cli /go/bin/ /usr/local/bin/ + +WORKDIR /app \ No newline at end of file From 6939c6ceaea918b73fd3f9b02d0b0e9bc6a0fb30 Mon Sep 17 00:00:00 2001 From: iris-zhang Date: Tue, 10 May 2022 20:28:17 -0700 Subject: [PATCH 309/376] Update dockerfile --- Dockerfile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 7813543f..964ebc77 100644 --- a/Dockerfile +++ b/Dockerfile @@ -46,4 +46,5 @@ RUN apt-get update -y && apt-get install -y \ # Copy all the binaries COPY --from=cli /go/bin/ /usr/local/bin/ -WORKDIR /app \ No newline at end of file +WORKDIR /app +ENTRYPOINT ["rosetta-cli"] \ No newline at end of file From 5ffd46203c7330e75254453fe87612abe0992dc0 Mon Sep 17 00:00:00 2001 From: raghavapamula Date: Tue, 10 May 2022 10:24:37 -0400 Subject: [PATCH 310/376] Added benchmarking for rosetta server endpoints through check:perf command Signed-off-by: raghavapamula Addressed Review Commments from Madhur and Rosie Signed-off-by: raghavapamula --- cmd/check_perf.go | 65 ++++++++++++ cmd/root.go | 3 + configuration/configuration.go | 32 ++++++ configuration/types.go | 37 ++++++- examples/configuration/default.json | 3 +- pkg/results/perf_results.go | 157 ++++++++++++++++++++++++++++ pkg/tester/benchmark_utils.go | 24 +++++ pkg/tester/data_perf.go | 96 +++++++++++++++++ 8 files changed, 414 insertions(+), 3 deletions(-) create mode 100644 cmd/check_perf.go create mode 100644 pkg/results/perf_results.go create mode 100644 pkg/tester/benchmark_utils.go create mode 100644 pkg/tester/data_perf.go diff --git a/cmd/check_perf.go b/cmd/check_perf.go new file mode 100644 index 00000000..613503af --- /dev/null +++ b/cmd/check_perf.go @@ -0,0 +1,65 @@ +// Copyright 2022 Coinbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package cmd + +import ( + "context" + "fmt" + "time" + + "github.com/coinbase/rosetta-cli/pkg/results" + t "github.com/coinbase/rosetta-cli/pkg/tester" + "github.com/spf13/cobra" + "golang.org/x/sync/errgroup" +) + +var ( + checkPerfCmd = &cobra.Command{ + Use: "check:perf", + Short: "Benchmark performance of time-critical endpoints of Asset Issuer's Rosetta Implementation", + Long: `This command can be used to benchmark the performance of time critical methods for a Rosetta server. +This is useful for ensuring that there are no performance degradations in the rosetta-server.`, + RunE: runCheckPerfCmd, + } +) + +func runCheckPerfCmd(_ *cobra.Command, _ []string) error { + ctx, cancel := context.WithCancel(Context) + defer cancel() + g, ctx := errgroup.WithContext(ctx) + + TotalNumEndpoints := int64(Config.Perf.NumTimesToHitEndpoints) * (Config.Perf.EndBlock - Config.Perf.StartBlock) + perfRawStats := &results.CheckPerfRawStats{AccountBalanceEndpointTotalTime: -1, BlockEndpointTotalTime: -1} + + fmt.Printf("Running Check:Perf for %s:%s for blocks %d-%d \n", Config.Network.Blockchain, Config.Network.Network, Config.Perf.StartBlock, Config.Perf.EndBlock) + + fetcher, timer, elapsed := t.SetupBenchmarking(Config) + blockEndpointTimeConstraint := time.Duration(Config.Perf.BlockEndpointTimeConstraintMs*TotalNumEndpoints) * time.Millisecond + blockEndpointCtx, blockEndpointCancel := context.WithTimeout(ctx, blockEndpointTimeConstraint) + g.Go(func() error { + return t.BmarkBlock(blockEndpointCtx, Config, fetcher, timer, elapsed, perfRawStats) + }) + defer blockEndpointCancel() + + fetcher, timer, elapsed = t.SetupBenchmarking(Config) + accountBalanceEndpointTimeConstraint := time.Duration(Config.Perf.AccountBalanceEndpointTimeConstraintMs*TotalNumEndpoints) * time.Millisecond + accountBalanceEndpointCtx, accountBalanceEndpointCancel := context.WithTimeout(ctx, accountBalanceEndpointTimeConstraint) + g.Go(func() error { + return t.BmarkAccountBalance(accountBalanceEndpointCtx, Config, fetcher, timer, elapsed, perfRawStats) + }) + defer accountBalanceEndpointCancel() + + return results.ExitPerf(Config.Perf, g.Wait(), perfRawStats) +} diff --git a/cmd/root.go b/cmd/root.go index 649bd2f6..1c41844d 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -240,6 +240,9 @@ default values.`, // Utils rootCmd.AddCommand(utilsAsserterConfigurationCmd) rootCmd.AddCommand(utilsTrainZstdCmd) + + // Benchmark commands + rootCmd.AddCommand(checkPerfCmd) } func initConfig() { diff --git a/configuration/configuration.go b/configuration/configuration.go index c92c2cd2..3b685a4e 100644 --- a/configuration/configuration.go +++ b/configuration/configuration.go @@ -44,6 +44,19 @@ func DefaultDataConfiguration() *DataConfiguration { } } +// DefaultPerfConfiguration returns the default *CheckPerfConfiguration +// for running `check:perf`. +func DefaultPerfConfiguration() *CheckPerfConfiguration { + return &CheckPerfConfiguration{ + StartBlock: 10, + BlockEndpointTimeConstraintMs: 50000000, + AccountBalanceEndpointTimeConstraintMs: 50000000, + EndBlock: 50, + NumTimesToHitEndpoints: 1, + StatsOutputFile: "./check_perf_stats.json", + } +} + // DefaultConfiguration returns a *Configuration with the // EthereumNetwork, DefaultURL, DefaultTimeout, // DefaultConstructionConfiguration and DefaultDataConfiguration. @@ -61,6 +74,24 @@ func DefaultConfiguration() *Configuration { } } +func populatePerfMissingFields( + perfConfig *CheckPerfConfiguration, +) *CheckPerfConfiguration { + if perfConfig == nil { + return nil + } + + if len(perfConfig.StatsOutputFile) == 0 { + perfConfig.StatsOutputFile = DefaultOutputFile + } + + if perfConfig.NumTimesToHitEndpoints == 0 { + perfConfig.NumTimesToHitEndpoints = DefaultNumTimesToHitEndpoints + } + + return perfConfig +} + func populateConstructionMissingFields( constructionConfig *ConstructionConfiguration, ) *ConstructionConfiguration { @@ -171,6 +202,7 @@ func populateMissingFields(config *Configuration) *Configuration { config.Construction = populateConstructionMissingFields(config.Construction) config.Data = populateDataMissingFields(config.Data) + config.Perf = populatePerfMissingFields(config.Perf) return config } diff --git a/configuration/types.go b/configuration/types.go index 1044143b..603afbef 100644 --- a/configuration/types.go +++ b/configuration/types.go @@ -61,6 +61,14 @@ const ( DefaultStatusPort = 9090 DefaultMaxReorgDepth = 100 + // Check Perf Default Configs + DefaultStartBlock = 100 + DefaultEndBlock = 10000 + DefaultNumTimesToHitEndpoints = 50 + DefaultOutputFile = "./check_perf_stats.json" + DefaultBlockEndpointTimeConstraintMs = 5000 + DefaultAccountBalanceEndpointTimeConstraintMs = 5000 + // ETH Defaults EthereumIDBlockchain = "Ethereum" EthereumIDNetwork = "Ropsten" @@ -227,7 +235,7 @@ type DataConfiguration struct { InactiveReconciliationConcurrency uint64 `json:"inactive_reconciliation_concurrency"` // InactiveReconciliationFrequency is the number of blocks to wait between - // inactive reconiliations on each account. + // inactive reconciliations on each account. InactiveReconciliationFrequency uint64 `json:"inactive_reconciliation_frequency"` // LogBlocks is a boolean indicating whether to log processed blocks. @@ -340,7 +348,7 @@ type DataConfiguration struct { } // Configuration contains all configuration settings for running -// check:data or check:construction. +// check:data, check:construction, or check:perf. type Configuration struct { // Network is the *types.NetworkIdentifier where transactions should // be constructed and where blocks should be synced to monitor @@ -433,4 +441,29 @@ type Configuration struct { Construction *ConstructionConfiguration `json:"construction"` Data *DataConfiguration `json:"data"` + Perf *CheckPerfConfiguration `json:"perf"` +} + +//********************// +// Check Perf configs // +//********************// +type CheckPerfConfiguration struct { + + // StartBlock is the starting block for running check:perf. + // If not provided, then this defaults to 0 (the genesis block) + StartBlock int64 `json:"start_block,omitempty"` + + BlockEndpointTimeConstraintMs int64 `json:"block_endpoint_time_constraint_ms"` + + AccountBalanceEndpointTimeConstraintMs int64 `json:"account_balance_endpoint_time_constraint_ms"` + + // EndBlock is the ending block for running check:perf. + // Must be provided when running check:perf + EndBlock int64 `json:"end_block"` + + // NumTimesToHitEndpoints is the number of times each rosetta-server endpoint will be benchmarked + NumTimesToHitEndpoints int `json:"num_times_to_hit_endpoints"` + + // Location to output test results + StatsOutputFile string `json:"check_perf_output_dir"` } diff --git a/examples/configuration/default.json b/examples/configuration/default.json index 11ee99f3..df5c68ab 100644 --- a/examples/configuration/default.json +++ b/examples/configuration/default.json @@ -39,5 +39,6 @@ "results_output_file": "", "pruning_disabled": false, "initial_balance_fetch_disabled": false - } + }, + "perf": null } \ No newline at end of file diff --git a/pkg/results/perf_results.go b/pkg/results/perf_results.go new file mode 100644 index 00000000..a0349c6e --- /dev/null +++ b/pkg/results/perf_results.go @@ -0,0 +1,157 @@ +// Copyright 2022 Coinbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package results + +import ( + "fmt" + "log" + "os" + "strconv" + "time" + + "github.com/coinbase/rosetta-cli/configuration" + "github.com/coinbase/rosetta-sdk-go/utils" + "github.com/olekukonko/tablewriter" +) + +// Output writes *CheckPerfResults to the provided +// path. +func (c *CheckPerfStats) Output(path string) { + if len(path) > 0 { + writeErr := utils.SerializeAndWrite(path, c) + if writeErr != nil { + log.Printf("%s: unable to save results\n", writeErr.Error()) + } + } +} + +type CheckPerfRawStats struct { + BlockEndpointTotalTime time.Duration + BlockEndpointNumErrors int64 + AccountBalanceEndpointTotalTime time.Duration + AccountBalanceNumErrors int64 +} + +// CheckPerfStats contains interesting stats that +// are counted while running the check:perf. +type CheckPerfStats struct { + StartBlock int64 `json:"start_block"` + EndBlock int64 `json:"end_block"` + NumTimesHitEachEndpoint int `json:"num_times_hit_each_endpoint"` + AccountBalanceEndpointAverageTimeMs int64 `json:"account_balance_endpoint_average_time_ms"` + AccountBalanceEndpointTotalTimeMs int64 `json:"account_balance_endpoint_total_time_ms"` + AccountBalanceEndpointNumErrors int64 `json:"account_balance_endpoint_num_errors"` + BlockEndpointAverageTimeMs int64 `json:"block_endpoint_average_time_ms"` + BlockEndpointTotalTimeMs int64 `json:"block_endpoint_total_time_ms"` + BlockEndpointNumErrors int64 `json:"block_endpoint_num_errors"` +} + +// Print logs CheckPerfStats to the console. +func (c *CheckPerfStats) Print() { + table := tablewriter.NewWriter(os.Stdout) + table.SetRowLine(true) + table.SetRowSeparator("-") + table.SetHeader([]string{"check:perf Stats", "Description", "Value"}) + table.Append([]string{"Start Block", "The Starting Block", strconv.FormatInt(c.StartBlock, 10)}) + table.Append([]string{"End Block", "The Ending Block", strconv.FormatInt(c.EndBlock, 10)}) + table.Append([]string{"Num Times Each Endpoint", "Number of times that each endpoint was hit", strconv.FormatInt(int64(c.NumTimesHitEachEndpoint), 10)}) + table.Append( + []string{ + "/Block Endpoint Total Time", + "Total elapsed time taken to fetch all blocks (ms)", + strconv.FormatInt(c.BlockEndpointTotalTimeMs, 10), + }, + ) + table.Append( + []string{ + "/Block Endpoint Average Time", + "Average time taken to fetch each block (ms)", + strconv.FormatInt(c.BlockEndpointAverageTimeMs, 10), + }, + ) + table.Append( + []string{ + "/Block Endpoint Num Errors", + "Total num errors occurred while fetching blocks", + strconv.FormatInt(c.BlockEndpointNumErrors, 10), + }, + ) + table.Append( + []string{ + "/Account/Balance Endpoint Average Time", + "Average time taken to fetch each account balance (ms)", + strconv.FormatInt(c.AccountBalanceEndpointAverageTimeMs, 10), + }, + ) + table.Append( + []string{ + "/Account/Balance Endpoint Total Time", + "Total elapsed time taken to fetch all account balances (ms)", + strconv.FormatInt(c.AccountBalanceEndpointTotalTimeMs, 10), + }, + ) + table.Append( + []string{ + "/Account/Balance Endpoint Num Errors", + "Total num errors occurred while fetching account balances", + strconv.FormatInt(c.AccountBalanceEndpointNumErrors, 10), + }, + ) + + table.Render() +} + +// ComputeCheckPerfStats returns a populated CheckPerfStats. +func ComputeCheckPerfStats( + config *configuration.CheckPerfConfiguration, + rawStats *CheckPerfRawStats, +) *CheckPerfStats { + totalNumEndpointsHit := (config.EndBlock - config.StartBlock) * int64(config.NumTimesToHitEndpoints) + stats := &CheckPerfStats{ + BlockEndpointAverageTimeMs: rawStats.BlockEndpointTotalTime.Milliseconds() / totalNumEndpointsHit, + BlockEndpointTotalTimeMs: rawStats.BlockEndpointTotalTime.Milliseconds(), + BlockEndpointNumErrors: rawStats.BlockEndpointNumErrors, + AccountBalanceEndpointAverageTimeMs: rawStats.AccountBalanceEndpointTotalTime.Milliseconds() / totalNumEndpointsHit, + AccountBalanceEndpointTotalTimeMs: rawStats.AccountBalanceEndpointTotalTime.Milliseconds(), + AccountBalanceEndpointNumErrors: rawStats.AccountBalanceNumErrors, + StartBlock: config.StartBlock, + EndBlock: config.EndBlock, + NumTimesHitEachEndpoint: config.NumTimesToHitEndpoints, + } + + return stats +} + +// ExitPerf exits check:perf, logs the test results to the console, +// and to a provided output path. +func ExitPerf( + config *configuration.CheckPerfConfiguration, + err error, + rawStats *CheckPerfRawStats, +) error { + if err != nil { + log.Fatal(fmt.Errorf("Check:Perf Failed!: %w", err)) + } + + stats := ComputeCheckPerfStats( + config, + rawStats, + ) + + stats.Print() + stats.Output(config.StatsOutputFile) + + return err +} diff --git a/pkg/tester/benchmark_utils.go b/pkg/tester/benchmark_utils.go new file mode 100644 index 00000000..eb64dbe5 --- /dev/null +++ b/pkg/tester/benchmark_utils.go @@ -0,0 +1,24 @@ +// Copyright 2022 Coinbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package tester + +import "time" + +func timerFactory() func() time.Duration { + start := time.Now() + return func() time.Duration { + return time.Since(start) + } +} diff --git a/pkg/tester/data_perf.go b/pkg/tester/data_perf.go new file mode 100644 index 00000000..0cdefc5c --- /dev/null +++ b/pkg/tester/data_perf.go @@ -0,0 +1,96 @@ +// Copyright 2022 Coinbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package tester + +import ( + "context" + "errors" + "time" + + "github.com/coinbase/rosetta-cli/configuration" + "github.com/coinbase/rosetta-cli/pkg/results" + "github.com/coinbase/rosetta-sdk-go/fetcher" + "github.com/coinbase/rosetta-sdk-go/types" +) + +func SetupBenchmarking(config *configuration.Configuration) (*fetcher.Fetcher, func() time.Duration, chan time.Duration) { + // Create a new fetcher + fetcher := fetcher.New( + config.OnlineURL, + fetcher.WithMaxRetries(0), + ) + timer := timerFactory() + elapsed := make(chan time.Duration, 1) + return fetcher, timer, elapsed +} + +// Benchmark the asset issuer's /block endpoint +func BmarkBlock(ctx context.Context, config *configuration.Configuration, fetcher *fetcher.Fetcher, timer func() time.Duration, elapsed chan time.Duration, rawStats *results.CheckPerfRawStats) error { + total_errors := 0 + go func() { + for m := config.Perf.StartBlock; m < config.Perf.EndBlock; m++ { + for n := 0; n < config.Perf.NumTimesToHitEndpoints; n++ { + partialBlockId := &types.PartialBlockIdentifier{ + Hash: nil, + Index: &m, + } + _, err := fetcher.Block(ctx, config.Network, partialBlockId) + if err != nil { + total_errors++ + } + } + } + elapsed <- timer() + }() + select { + case <-ctx.Done(): + return errors.New("/block endpoint benchmarking timed out") + case timeTaken := <-elapsed: + rawStats.BlockEndpointTotalTime = timeTaken + rawStats.BlockEndpointNumErrors = int64(total_errors) + return nil + } +} + +// Benchmark the asset issuers /account/balance endpoint +func BmarkAccountBalance(ctx context.Context, config *configuration.Configuration, fetcher *fetcher.Fetcher, timer func() time.Duration, elapsed chan time.Duration, rawStats *results.CheckPerfRawStats) error { + total_errors := 0 + go func() { + for m := config.Perf.StartBlock; m < config.Perf.EndBlock; m++ { + for n := 0; n < config.Perf.NumTimesToHitEndpoints; n++ { + account := &types.AccountIdentifier{ + Address: "address", + } + partialBlockId := &types.PartialBlockIdentifier{ + Hash: nil, + Index: &m, + } + _, _, _, err := fetcher.AccountBalance(ctx, config.Network, account, partialBlockId, nil) + if err != nil { + total_errors++ + } + } + } + elapsed <- timer() + }() + select { + case <-ctx.Done(): + return errors.New("/account/balance endpoint benchmarking timed out") + case timeTaken := <-elapsed: + rawStats.AccountBalanceEndpointTotalTime = timeTaken + rawStats.AccountBalanceNumErrors = int64(total_errors) + return nil + } +} From b78b47e42fd84dab8f1495776959ccdac83a82f1 Mon Sep 17 00:00:00 2001 From: iris-zhang Date: Wed, 11 May 2022 12:03:25 -0700 Subject: [PATCH 311/376] Update readme --- README.md | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/README.md b/README.md index 7e50c0be..683b6209 100644 --- a/README.md +++ b/README.md @@ -35,6 +35,17 @@ To download the binary into a specific directory, run: curl -sSfL https://raw.githubusercontent.com/coinbase/rosetta-cli/master/scripts/install.sh | sh -s -- -b ``` +### Run via Docker +Running the following command will start a Docker container and present CLI for the Rosetta API. +``` +docker run -it [image-name] [command] +``` + +Example: To validate that the Data API implementation is correct, running the following command will start a Docker container with a data directory at ``. +``` +docker run -v "$(pwd):/data" -it [image-name] check:data --configuration-file /data/config.json +``` + ## Updates and Releases We recommend that you continually update your installation to the latest release as soon as possible. The latest release notes are available in our [Community](https://community.rosetta-api.org) board under the [Release](https://community.rosetta-api.org/c/releases/13) category. From 0b6eacaee2638b9a2385f70ac23fcbed8d1734a6 Mon Sep 17 00:00:00 2001 From: raghavapamula Date: Wed, 11 May 2022 15:16:31 -0400 Subject: [PATCH 312/376] Added error categorization for check:data --- cmd/check_data.go | 7 +++++-- pkg/errors/errors.go | 8 ++++++-- pkg/tester/data.go | 29 ++++++++++++++--------------- 3 files changed, 25 insertions(+), 19 deletions(-) diff --git a/cmd/check_data.go b/cmd/check_data.go index 9e6d0e7e..eb79d77c 100644 --- a/cmd/check_data.go +++ b/cmd/check_data.go @@ -17,11 +17,11 @@ package cmd import ( "context" "fmt" + "github.com/coinbase/rosetta-cli/pkg/errors" "time" "github.com/coinbase/rosetta-cli/pkg/results" "github.com/coinbase/rosetta-cli/pkg/tester" - "github.com/coinbase/rosetta-sdk-go/fetcher" "github.com/coinbase/rosetta-sdk-go/utils" "github.com/spf13/cobra" @@ -131,7 +131,7 @@ func runCheckDataCmd(_ *cobra.Command, _ []string) error { } } - dataTester := tester.InitializeData( + dataTester, err := tester.InitializeData( ctx, Config, Config.Network, @@ -142,6 +142,9 @@ func runCheckDataCmd(_ *cobra.Command, _ []string) error { &SignalReceived, ) + if err != nil { + return fmt.Errorf("%s:%s", errors.ErrInitDataTester, err) + } defer dataTester.CloseDatabase(ctx) g, ctx := errgroup.WithContext(ctx) diff --git a/pkg/errors/errors.go b/pkg/errors/errors.go index cac440e4..a5f93b93 100644 --- a/pkg/errors/errors.go +++ b/pkg/errors/errors.go @@ -21,6 +21,12 @@ import ( // Configuration Errors var ( + // Data Errors + + ErrCheckStorageTipFailed = errors.New("unable to check storage tip") + ErrDataCheckHalt = errors.New("data check halted") + ErrInitDataTester = errors.New("unexpected error occurred while trying to initialize data tester") + // Construction Configuration Errors ErrParseFileFailed = errors.New("unable to parse config files") @@ -30,8 +36,6 @@ var ( ErrParseWorkflowFailed = errors.New("unable to parse workflow") ErrConstructionConfigMissing = errors.New("construction configuration is missing") - // TODO: Data Configuration Errors - // Construction check errors ErrConstructionCheckHalt = errors.New("construction check halted") diff --git a/pkg/tester/data.go b/pkg/tester/data.go index 382c644f..e74e9bdc 100644 --- a/pkg/tester/data.go +++ b/pkg/tester/data.go @@ -19,6 +19,7 @@ import ( "encoding/json" "errors" "fmt" + customErrs "github.com/coinbase/rosetta-cli/pkg/errors" "log" "math/big" "net/http" @@ -28,7 +29,6 @@ import ( "github.com/coinbase/rosetta-cli/pkg/logger" "github.com/coinbase/rosetta-cli/pkg/processor" "github.com/coinbase/rosetta-cli/pkg/results" - "github.com/coinbase/rosetta-sdk-go/fetcher" "github.com/coinbase/rosetta-sdk-go/parser" "github.com/coinbase/rosetta-sdk-go/reconciler" @@ -145,10 +145,10 @@ func InitializeData( genesisBlock *types.BlockIdentifier, interestingAccount *types.AccountCurrency, signalReceived *bool, -) *DataTester { +) (*DataTester, error) { dataPath, err := utils.CreateCommandPath(config.DataDirectory, dataCmdName, network) if err != nil { - log.Fatalf("%s: cannot create command path", err.Error()) + return nil, fmt.Errorf("%s: cannot create command path", err.Error()) } opts := []database.BadgerOption{} @@ -164,17 +164,17 @@ func InitializeData( localStore, err := database.NewBadgerDatabase(ctx, dataPath, opts...) if err != nil { - log.Fatalf("%s: unable to initialize database", err.Error()) + return nil, fmt.Errorf("%s: unable to initialize database", err.Error()) } exemptAccounts, err := loadAccounts(config.Data.ExemptAccounts) if err != nil { - log.Fatalf("%s: unable to load exempt accounts", err.Error()) + return nil, fmt.Errorf("%s: unable to load exempt accounts", err.Error()) } interestingAccounts, err := loadAccounts(config.Data.InterestingAccounts) if err != nil { - log.Fatalf("%s: unable to load interesting accounts", err.Error()) + return nil, fmt.Errorf("%s: unable to load interesting accounts", err.Error()) } counterStorage := modules.NewCounterStorage(localStore) @@ -191,8 +191,7 @@ func InitializeData( network, ) if err != nil { - log.Fatalf(fmt.Sprintf("unable to initialize logger with error: %s", err.Error())) - return nil + return nil, fmt.Errorf("unable to initialize logger with error: %s", err.Error()) } var forceInactiveReconciliation bool @@ -216,16 +215,16 @@ func InitializeData( // Get all previously seen accounts seenAccounts, err := balanceStorage.GetAllAccountCurrency(ctx) if err != nil { - log.Fatalf("%s: unable to get previously seen accounts", err.Error()) + return nil, fmt.Errorf("%s: unable to get previously seen accounts", err.Error()) } networkOptions, fetchErr := fetcher.NetworkOptionsRetry(ctx, network, nil) - if err != nil { + if fetchErr != nil { log.Fatalf("%s: unable to get network options", fetchErr.Err.Error()) } if len(networkOptions.Allow.BalanceExemptions) > 0 && config.Data.InitialBalanceFetchDisabled { - log.Fatal("found balance exemptions but initial balance fetch disabled") + return nil, fmt.Errorf("found balance exemptions but initial balance fetch disabled") } parser := parser.New( @@ -308,10 +307,10 @@ func InitializeData( genesisBlock, ) if err != nil { - log.Fatalf("%s: unable to bootstrap balances", err.Error()) + return nil, fmt.Errorf("%s: unable to bootstrap balances", err.Error()) } case err != nil: - log.Fatalf("%s: unable to get head block identifier", err.Error()) + return nil, fmt.Errorf("%s: unable to get head block identifier", err.Error()) default: log.Println("Skipping balance bootstrapping because already started syncing") } @@ -368,7 +367,7 @@ func InitializeData( historicalBalanceEnabled: historicalBalanceEnabled, parser: parser, forceInactiveReconciliation: &forceInactiveReconciliation, - } + }, nil } // StartSyncing syncs from startIndex to endIndex. @@ -860,7 +859,7 @@ func (t *DataTester) HandleErr(err error, sigListeners *[]context.CancelFunc) er t.config, t.counterStorage, t.balanceStorage, - errors.New("check halted"), + fmt.Errorf("%w: %v", customErrs.ErrDataCheckHalt, err.Error()), "", "", ) From 9973f3cd946a920fe42989f37492a6177e78a2a7 Mon Sep 17 00:00:00 2001 From: iris-zhang Date: Wed, 11 May 2022 14:39:09 -0700 Subject: [PATCH 313/376] Add node url into command flag --- cmd/check_construction.go | 2 ++ cmd/check_data.go | 3 ++- cmd/root.go | 23 +++++++++++++++++++++++ 3 files changed, 27 insertions(+), 1 deletion(-) diff --git a/cmd/check_construction.go b/cmd/check_construction.go index 617a4a0d..a34993b6 100644 --- a/cmd/check_construction.go +++ b/cmd/check_construction.go @@ -18,6 +18,7 @@ import ( "context" "fmt" "github.com/coinbase/rosetta-cli/pkg/errors" + "log" "time" "github.com/coinbase/rosetta-cli/pkg/results" @@ -76,6 +77,7 @@ func runCheckConstructionCmd(_ *cobra.Command, _ []string) error { fetcherOpts = append(fetcherOpts, fetcher.WithForceRetry()) } + log.Println("node url: ", Config.OnlineURL) fetcher := fetcher.New( Config.OnlineURL, fetcherOpts..., diff --git a/cmd/check_data.go b/cmd/check_data.go index eb79d77c..9b7bd733 100644 --- a/cmd/check_data.go +++ b/cmd/check_data.go @@ -18,6 +18,7 @@ import ( "context" "fmt" "github.com/coinbase/rosetta-cli/pkg/errors" + "log" "time" "github.com/coinbase/rosetta-cli/pkg/results" @@ -83,7 +84,7 @@ func runCheckDataCmd(_ *cobra.Command, _ []string) error { if Config.ForceRetry { fetcherOpts = append(fetcherOpts, fetcher.WithForceRetry()) } - + log.Println("node url: ", Config.OnlineURL) fetcher := fetcher.New( Config.OnlineURL, fetcherOpts..., diff --git a/cmd/root.go b/cmd/root.go index 1c41844d..ff30ed49 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -48,6 +48,7 @@ var ( cpuProfile string memProfile string blockProfile string + nodeUrl string // Config is the populated *configuration.Configuration from // the configurationFile. If none is provided, this is set @@ -217,6 +218,14 @@ default values.`, "", // Default to skip validation `Check that /network/options matches contents of file at this path`, ) + + checkDataCmd.Flags().StringVar( + &nodeUrl, + "node-url", + "", + "Override node url in configuration file", + ) + rootCmd.AddCommand(checkDataCmd) checkConstructionCmd.Flags().StringVar( &asserterConfigurationFile, @@ -224,6 +233,14 @@ default values.`, "", // Default to skip validation `Check that /network/options matches contents of file at this path`, ) + + checkConstructionCmd.Flags().StringVar( + &nodeUrl, + "node-url", + "", + "Override node url in configuration file", + ) + rootCmd.AddCommand(checkConstructionCmd) // View Commands @@ -260,9 +277,15 @@ func initConfig() { } else { Config, err = configuration.LoadConfiguration(Context, configurationFile) } + if err != nil { log.Fatalf("%s: unable to load configuration", err.Error()) } + + // Override node url in configuration file when it's explicitly set via CLI + if len(nodeUrl) != 0 { + Config.OnlineURL = nodeUrl + } } func ensureDataDirectoryExists() { From 93c22bce482512986a71cb090bd35e24cd95f120 Mon Sep 17 00:00:00 2001 From: iris-zhang Date: Wed, 11 May 2022 15:47:36 -0700 Subject: [PATCH 314/376] Add offline node url as well --- cmd/check_construction.go | 2 -- cmd/check_data.go | 3 +-- cmd/root.go | 29 ++++++++++++++++++++--------- 3 files changed, 21 insertions(+), 13 deletions(-) diff --git a/cmd/check_construction.go b/cmd/check_construction.go index a34993b6..617a4a0d 100644 --- a/cmd/check_construction.go +++ b/cmd/check_construction.go @@ -18,7 +18,6 @@ import ( "context" "fmt" "github.com/coinbase/rosetta-cli/pkg/errors" - "log" "time" "github.com/coinbase/rosetta-cli/pkg/results" @@ -77,7 +76,6 @@ func runCheckConstructionCmd(_ *cobra.Command, _ []string) error { fetcherOpts = append(fetcherOpts, fetcher.WithForceRetry()) } - log.Println("node url: ", Config.OnlineURL) fetcher := fetcher.New( Config.OnlineURL, fetcherOpts..., diff --git a/cmd/check_data.go b/cmd/check_data.go index 9b7bd733..eb79d77c 100644 --- a/cmd/check_data.go +++ b/cmd/check_data.go @@ -18,7 +18,6 @@ import ( "context" "fmt" "github.com/coinbase/rosetta-cli/pkg/errors" - "log" "time" "github.com/coinbase/rosetta-cli/pkg/results" @@ -84,7 +83,7 @@ func runCheckDataCmd(_ *cobra.Command, _ []string) error { if Config.ForceRetry { fetcherOpts = append(fetcherOpts, fetcher.WithForceRetry()) } - log.Println("node url: ", Config.OnlineURL) + fetcher := fetcher.New( Config.OnlineURL, fetcherOpts..., diff --git a/cmd/root.go b/cmd/root.go index ff30ed49..93cb90b6 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -48,7 +48,8 @@ var ( cpuProfile string memProfile string blockProfile string - nodeUrl string + onlineUrl string + offlineUrl string // Config is the populated *configuration.Configuration from // the configurationFile. If none is provided, this is set @@ -220,10 +221,10 @@ default values.`, ) checkDataCmd.Flags().StringVar( - &nodeUrl, - "node-url", + &onlineUrl, + "online-url", "", - "Override node url in configuration file", + "Override online node url in configuration file", ) rootCmd.AddCommand(checkDataCmd) @@ -235,10 +236,17 @@ default values.`, ) checkConstructionCmd.Flags().StringVar( - &nodeUrl, - "node-url", + &onlineUrl, + "online-url", "", - "Override node url in configuration file", + "Override online node url in configuration file", + ) + + checkConstructionCmd.Flags().StringVar( + &offlineUrl, + "offline-url", + "", + "Override offline node url in configuration file", ) rootCmd.AddCommand(checkConstructionCmd) @@ -283,8 +291,11 @@ func initConfig() { } // Override node url in configuration file when it's explicitly set via CLI - if len(nodeUrl) != 0 { - Config.OnlineURL = nodeUrl + if len(onlineUrl) != 0 { + Config.OnlineURL = onlineUrl + } + if len(offlineUrl) != 0 { + Config.Construction.OfflineURL = offlineUrl } } From 7b1d65ca92d91348427f2dea69555f717ff73b0c Mon Sep 17 00:00:00 2001 From: iris-zhang Date: Mon, 16 May 2022 19:57:01 -0700 Subject: [PATCH 315/376] Add start and end index --- cmd/root.go | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/cmd/root.go b/cmd/root.go index 93cb90b6..b600a583 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -50,6 +50,8 @@ var ( blockProfile string onlineUrl string offlineUrl string + startIndex int64 + endIndex int64 // Config is the populated *configuration.Configuration from // the configurationFile. If none is provided, this is set @@ -227,6 +229,20 @@ default values.`, "Override online node url in configuration file", ) + checkDataCmd.Flags().Int64Var( + &startIndex, + "start-block", + -1, + `start-block is the block height to start syncing from. This will override the start_index from configuration file`, + ) + + checkDataCmd.Flags().Int64Var( + &endIndex, + "end-block", + -1, + `End-block configures the syncer to stop once reaching a particular block height. This will override the index from configuration file`, + ) + rootCmd.AddCommand(checkDataCmd) checkConstructionCmd.Flags().StringVar( &asserterConfigurationFile, @@ -297,6 +313,15 @@ func initConfig() { if len(offlineUrl) != 0 { Config.Construction.OfflineURL = offlineUrl } + + // Override start and end syncing index in configuration file when it's explicitly set via CLI + if startIndex != -1 { + Config.Data.StartIndex = &startIndex + } + + if endIndex != -1 { + Config.Data.EndConditions.Index = &endIndex + } } func ensureDataDirectoryExists() { From c24e663e3a1f564e3291120035ff741fc2c6268d Mon Sep 17 00:00:00 2001 From: iris-zhang Date: Mon, 16 May 2022 20:12:50 -0700 Subject: [PATCH 316/376] fix lint --- cmd/root.go | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/cmd/root.go b/cmd/root.go index b600a583..22907a6b 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -48,10 +48,10 @@ var ( cpuProfile string memProfile string blockProfile string - onlineUrl string - offlineUrl string - startIndex int64 - endIndex int64 + onlineURL string + offlineURL string + startIndex int64 + endIndex int64 // Config is the populated *configuration.Configuration from // the configurationFile. If none is provided, this is set @@ -223,7 +223,7 @@ default values.`, ) checkDataCmd.Flags().StringVar( - &onlineUrl, + &onlineURL, "online-url", "", "Override online node url in configuration file", @@ -252,14 +252,14 @@ default values.`, ) checkConstructionCmd.Flags().StringVar( - &onlineUrl, + &onlineURL, "online-url", "", "Override online node url in configuration file", ) checkConstructionCmd.Flags().StringVar( - &offlineUrl, + &offlineURL, "offline-url", "", "Override offline node url in configuration file", @@ -307,11 +307,11 @@ func initConfig() { } // Override node url in configuration file when it's explicitly set via CLI - if len(onlineUrl) != 0 { - Config.OnlineURL = onlineUrl + if len(onlineURL) != 0 { + Config.OnlineURL = onlineURL } - if len(offlineUrl) != 0 { - Config.Construction.OfflineURL = offlineUrl + if len(offlineURL) != 0 { + Config.Construction.OfflineURL = offlineURL } // Override start and end syncing index in configuration file when it's explicitly set via CLI From 6f56d03d235a54d4bde7a3a6b106531807059221 Mon Sep 17 00:00:00 2001 From: Madhur Shrimal Date: Thu, 19 May 2022 14:46:50 -0700 Subject: [PATCH 317/376] release instructions (#314) --- Dockerfile | 2 +- README.md | 10 ++++++---- cmd/root.go | 2 +- 3 files changed, 8 insertions(+), 6 deletions(-) diff --git a/Dockerfile b/Dockerfile index 964ebc77..396713c6 100644 --- a/Dockerfile +++ b/Dockerfile @@ -31,7 +31,7 @@ RUN mkdir -p "$GOPATH/src" "$GOPATH/bin" && chmod -R 777 "$GOPATH" WORKDIR /go/src -ARG VERSION=v0.7.6 +ARG VERSION=v0.7.7 RUN git clone https://github.com/coinbase/rosetta-cli.git && \ cd rosetta-cli && \ git fetch --all --tags && \ diff --git a/README.md b/README.md index 683b6209..1c3d68f8 100644 --- a/README.md +++ b/README.md @@ -93,6 +93,11 @@ If you are developing on both the `rosetta-cli` and `rosetta-sdk-go` repositorie ``` replace "github.com/coinbase/rosetta-sdk-go" v0.6.8 => "../rosetta-sdk-go" ``` +## Release +* When we release a new rosetta-cli version, please update the [version number](https://github.com/coinbase/rosetta-cli/blob/master/cmd/root.go#L297) so that `rosetta-cli version` command can print the correct value. +* Create binaries and upload all the binaries in the new release tag (Eg: https://github.com/coinbase/rosetta-cli/releases/tag/v0.7.7) + * `make compile version=` + * All the binaries will be created in the `bin` folder and should have extension as `tar.gz` and new version number ### Helper/Handler Many of the packages use a `Helper/Handler` interface pattern to acquire required information or to send events to some client implementation. An example of this is in the `reconciler` package where a `Helper` is used to get the account balance and the `Handler` is called to indicate whether the reconciliation of an account was successful. @@ -128,10 +133,7 @@ To help you with examples, we developed complete Rosetta API reference implement You can also find community implementations for a variety of blockchains in the [rosetta-ecosystem](https://github.com/coinbase/rosetta-ecosystem) repository, and in the [ecosystem category](https://community.rosetta-api.org/c/ecosystem) of our community site. + ## License This project is available open source under the terms of the [Apache 2.0 License](https://opensource.org/licenses/Apache-2.0). - -## Release -When we release a new rosetta-cli version, please update the [version number](https://github.com/coinbase/rosetta-cli/blob/master/cmd/root.go#L297) so that `rosetta-cli version` command can print the correct value. - © 2022 Coinbase \ No newline at end of file diff --git a/cmd/root.go b/cmd/root.go index 1c41844d..735e64d2 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -298,6 +298,6 @@ var versionCmd = &cobra.Command{ Use: "version", Short: "Print rosetta-cli version", Run: func(cmd *cobra.Command, args []string) { - fmt.Println("v0.7.6") + fmt.Println("v0.7.7") }, } From 342cb05ef923ff07fb52f5e5988185bc71f78b85 Mon Sep 17 00:00:00 2001 From: iris-zhang Date: Wed, 18 May 2022 14:26:50 -0700 Subject: [PATCH 318/376] set InitialBalanceFetchDisabled when index is set --- cmd/root.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/cmd/root.go b/cmd/root.go index 22907a6b..98045ce6 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -317,6 +317,9 @@ func initConfig() { // Override start and end syncing index in configuration file when it's explicitly set via CLI if startIndex != -1 { Config.Data.StartIndex = &startIndex + // Configures rosetta-cli to lookup the balance of newly seen accounts at the + // parent block before applying operations. Otherwise the balance will be 0. + Config.Data.InitialBalanceFetchDisabled = false } if endIndex != -1 { From c725feabfc2a05d854682f8b92171aec054fd72a Mon Sep 17 00:00:00 2001 From: iris-zhang Date: Fri, 20 May 2022 12:23:13 -0700 Subject: [PATCH 319/376] Add command flag for result file --- Dockerfile | 2 +- cmd/root.go | 41 ++++++++++++++++++++++++++++++++--------- 2 files changed, 33 insertions(+), 10 deletions(-) diff --git a/Dockerfile b/Dockerfile index 396713c6..4840968b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -31,7 +31,7 @@ RUN mkdir -p "$GOPATH/src" "$GOPATH/bin" && chmod -R 777 "$GOPATH" WORKDIR /go/src -ARG VERSION=v0.7.7 +ARG VERSION=v0.7.9 RUN git clone https://github.com/coinbase/rosetta-cli.git && \ cd rosetta-cli && \ git fetch --all --tags && \ diff --git a/cmd/root.go b/cmd/root.go index 9c82db8a..0291ee33 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -44,14 +44,16 @@ var ( PersistentPreRunE: rootPreRun, } - configurationFile string - cpuProfile string - memProfile string - blockProfile string - onlineURL string - offlineURL string - startIndex int64 - endIndex int64 + configurationFile string + cpuProfile string + memProfile string + blockProfile string + onlineURL string + offlineURL string + startIndex int64 + endIndex int64 + dataResultFile string + constructionResultFile string // Config is the populated *configuration.Configuration from // the configurationFile. If none is provided, this is set @@ -243,6 +245,13 @@ default values.`, `End-block configures the syncer to stop once reaching a particular block height. This will override the index from configuration file`, ) + checkDataCmd.Flags().StringVar( + &dataResultFile, + "result-file", + "", + "Result-file configures the location of validation result. This will override the results_output_file from configuration file", + ) + rootCmd.AddCommand(checkDataCmd) checkConstructionCmd.Flags().StringVar( &asserterConfigurationFile, @@ -265,6 +274,12 @@ default values.`, "Override offline node url in configuration file", ) + checkConstructionCmd.Flags().StringVar( + &constructionResultFile, + "result-file", + "", + "Result-file configures the location of validation result. This will override the results_output_file from configuration file", + ) rootCmd.AddCommand(checkConstructionCmd) // View Commands @@ -325,6 +340,14 @@ func initConfig() { if endIndex != -1 { Config.Data.EndConditions.Index = &endIndex } + + if len(dataResultFile) != 0 { + Config.Data.ResultsOutputFile = dataResultFile + } + + if len(constructionResultFile) != 0 { + Config.Construction.ResultsOutputFile = constructionResultFile + } } func ensureDataDirectoryExists() { @@ -360,6 +383,6 @@ var versionCmd = &cobra.Command{ Use: "version", Short: "Print rosetta-cli version", Run: func(cmd *cobra.Command, args []string) { - fmt.Println("v0.7.7") + fmt.Println("v0.7.9") }, } From 0de1e426a545eec5fbf963a875f27e500aeee82c Mon Sep 17 00:00:00 2001 From: irisZhangCB <98374544+irisZhangCB@users.noreply.github.com> Date: Fri, 20 May 2022 14:34:21 -0700 Subject: [PATCH 320/376] Bump version (#318) --- Dockerfile | 2 +- cmd/root.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 4840968b..edd3b8cc 100644 --- a/Dockerfile +++ b/Dockerfile @@ -31,7 +31,7 @@ RUN mkdir -p "$GOPATH/src" "$GOPATH/bin" && chmod -R 777 "$GOPATH" WORKDIR /go/src -ARG VERSION=v0.7.9 +ARG VERSION=v0.7.8 RUN git clone https://github.com/coinbase/rosetta-cli.git && \ cd rosetta-cli && \ git fetch --all --tags && \ diff --git a/cmd/root.go b/cmd/root.go index 0291ee33..3772530d 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -383,6 +383,6 @@ var versionCmd = &cobra.Command{ Use: "version", Short: "Print rosetta-cli version", Run: func(cmd *cobra.Command, args []string) { - fmt.Println("v0.7.9") + fmt.Println("v0.7.8") }, } From d1e45147e5a6248939fb878e8fec115d7112a6be Mon Sep 17 00:00:00 2001 From: irisZhangCB <98374544+irisZhangCB@users.noreply.github.com> Date: Wed, 25 May 2022 11:42:47 -0700 Subject: [PATCH 321/376] Add datadir to Command param (#320) * Bump version * Add datadir to Command param --- cmd/root.go | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/cmd/root.go b/cmd/root.go index 3772530d..eda76cba 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -54,6 +54,7 @@ var ( endIndex int64 dataResultFile string constructionResultFile string + dataDirectory string // Config is the populated *configuration.Configuration from // the configurationFile. If none is provided, this is set @@ -252,6 +253,12 @@ default values.`, "Result-file configures the location of validation result. This will override the results_output_file from configuration file", ) + checkDataCmd.Flags().StringVar( + &dataDirectory, + "data-dir", + "", + "Data-dir configures the location of logs and data for validation. This will override the data_directory from configuration file", + ) rootCmd.AddCommand(checkDataCmd) checkConstructionCmd.Flags().StringVar( &asserterConfigurationFile, @@ -348,6 +355,10 @@ func initConfig() { if len(constructionResultFile) != 0 { Config.Construction.ResultsOutputFile = constructionResultFile } + + if len(dataDirectory) != 0 { + Config.DataDirectory = dataDirectory + } } func ensureDataDirectoryExists() { From 0b387f58a447e94afff4487fd7236ad6dd5884d1 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 1 Jun 2022 22:16:10 -0700 Subject: [PATCH 322/376] Bump github.com/spf13/cobra from 1.3.0 to 1.4.0 (#282) Bumps [github.com/spf13/cobra](https://github.com/spf13/cobra) from 1.3.0 to 1.4.0. - [Release notes](https://github.com/spf13/cobra/releases) - [Changelog](https://github.com/spf13/cobra/blob/master/CHANGELOG.md) - [Commits](https://github.com/spf13/cobra/compare/v1.3.0...v1.4.0) --- updated-dependencies: - dependency-name: github.com/spf13/cobra dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 8 +- go.sum | 423 +-------------------------------------------------------- 2 files changed, 9 insertions(+), 422 deletions(-) diff --git a/go.mod b/go.mod index f572db24..e9271db5 100644 --- a/go.mod +++ b/go.mod @@ -5,10 +5,16 @@ go 1.16 require ( github.com/coinbase/rosetta-sdk-go v0.7.9 github.com/fatih/color v1.13.0 + github.com/golang/protobuf v1.5.2 // indirect + github.com/google/go-cmp v0.5.6 // indirect + github.com/mattn/go-colorable v0.1.12 // indirect github.com/olekukonko/tablewriter v0.0.5 github.com/pkg/errors v0.9.1 - github.com/spf13/cobra v1.3.0 + github.com/spf13/cobra v1.4.0 github.com/stretchr/testify v1.7.1 go.uber.org/zap v1.21.0 + golang.org/x/crypto v0.0.0-20210817164053-32db794688a5 // indirect golang.org/x/sync v0.0.0-20210220032951-036812b2e83c + golang.org/x/sys v0.0.0-20211205182925-97ca703d548d // indirect + google.golang.org/protobuf v1.27.1 // indirect ) diff --git a/go.sum b/go.sum index cebc8547..314f143a 100644 --- a/go.sum +++ b/go.sum @@ -8,46 +8,14 @@ cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTj cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0= cloud.google.com/go v0.50.0/go.mod h1:r9sluTvynVuxRIOHXQEHMFffphuXHOMZMycpNR5e6To= cloud.google.com/go v0.51.0/go.mod h1:hWtGJ6gnXH+KgDv+V0zFGDvpi07n3z8ZNj3T1RW0Gcw= -cloud.google.com/go v0.52.0/go.mod h1:pXajvRH/6o3+F9jDHZWQ5PbGhn+o8w9qiu/CffaVdO4= -cloud.google.com/go v0.53.0/go.mod h1:fp/UouUEsRkN6ryDKNW/Upv/JBKnv6WDthjR6+vze6M= -cloud.google.com/go v0.54.0/go.mod h1:1rq2OEkV3YMf6n/9ZvGWI3GWw0VoqH/1x2nd8Is/bPc= -cloud.google.com/go v0.56.0/go.mod h1:jr7tqZxxKOVYizybht9+26Z/gUq7tiRzu+ACVAMbKVk= -cloud.google.com/go v0.57.0/go.mod h1:oXiQ6Rzq3RAkkY7N6t3TcE6jE+CIBBbA36lwQ1JyzZs= -cloud.google.com/go v0.62.0/go.mod h1:jmCYTdRCQuc1PHIIJ/maLInMho30T/Y0M4hTdTShOYc= -cloud.google.com/go v0.65.0/go.mod h1:O5N8zS7uWy9vkA9vayVHs65eM1ubvY4h553ofrNHObY= -cloud.google.com/go v0.72.0/go.mod h1:M+5Vjvlc2wnp6tjzE102Dw08nGShTscUx2nZMufOKPI= -cloud.google.com/go v0.74.0/go.mod h1:VV1xSbzvo+9QJOxLDaJfTjx5e+MePCpCWwvftOeQmWk= -cloud.google.com/go v0.78.0/go.mod h1:QjdrLG0uq+YwhjoVOLsS1t7TW8fs36kLs4XO5R5ECHg= -cloud.google.com/go v0.79.0/go.mod h1:3bzgcEeQlzbuEAYu4mrWhKqWjmpprinYgKJLgKHnbb8= -cloud.google.com/go v0.81.0/go.mod h1:mk/AM35KwGk/Nm2YSeZbxXdrNK3KZOYHmLkOqC2V6E0= -cloud.google.com/go v0.83.0/go.mod h1:Z7MJUsANfY0pYPdw0lbnivPx4/vhy/e2FEkSkF7vAVY= -cloud.google.com/go v0.84.0/go.mod h1:RazrYuxIK6Kb7YrzzhPoLmCVzl7Sup4NrbKPg8KHSUM= -cloud.google.com/go v0.87.0/go.mod h1:TpDYlFy7vuLzZMMZ+B6iRiELaY7z/gJPaqbMx6mlWcY= -cloud.google.com/go v0.90.0/go.mod h1:kRX0mNRHe0e2rC6oNakvwQqzyDmg57xJ+SZU1eT2aDQ= -cloud.google.com/go v0.93.3/go.mod h1:8utlLll2EF5XMAV15woO4lSbWQlk8rer9aLOfLh7+YI= -cloud.google.com/go v0.94.1/go.mod h1:qAlAugsXlC+JWO+Bke5vCtc9ONxjQT3drlTTnAplMW4= -cloud.google.com/go v0.97.0/go.mod h1:GF7l59pYBVlXQIBLx3a761cZ41F9bBH3JUlihCt2Udc= -cloud.google.com/go v0.98.0/go.mod h1:ua6Ush4NALrHk5QXDWnjvZHN93OuF0HfuEPq9I1X0cM= -cloud.google.com/go v0.99.0/go.mod h1:w0Xx2nLzqWJPuozYQX+hFfCSI8WioryfRDzkoI/Y2ZA= cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= -cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc= -cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg= -cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc= -cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ= cloud.google.com/go/bigtable v1.2.0/go.mod h1:JcVAOl45lrTmQfLj7T6TxyMzIN/3FGGcFm+2xVAli2o= cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= -cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= -cloud.google.com/go/firestore v1.6.1/go.mod h1:asNXNOzBdyVQmEU+ggO8UPodTkEVFW5Qx+rwHnAz+EY= cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= -cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA= -cloud.google.com/go/pubsub v1.3.1/go.mod h1:i+ucay31+CNRpDW4Lu78I4xXG+O1r/MAHgjpRVR+TSU= cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos= -cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk= -cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs= -cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= collectd.org v0.3.0/go.mod h1:A/8DzQBkF6abtvrT2j/AU/4tiBgJWYyh0y/oB/4MlWE= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= filippo.io/edwards25519 v1.0.0-rc.1 h1:m0VOOB23frXZvAOK44usCgLWvtsxIoMCTBGJZlpmGfU= @@ -60,7 +28,6 @@ github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v0.3.0/go.mod h1:tPaiy8S5bQ github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/DATA-DOG/go-sqlmock v1.3.3/go.mod h1:f/Ixk793poVmq4qj/V1dPUg2JEAKC73Q5eFN3EC/SaM= -github.com/DataDog/datadog-go v3.2.0+incompatible/go.mod h1:LButxg5PwREeZtORoXG3tL4fMGNddJ+vMq1mwgfaqoQ= github.com/DataDog/zstd v1.5.0 h1:+K/VEwIAaPcHiMtQvpLD4lqW7f0Gk3xdYZmI1hD+CXo= github.com/DataDog/zstd v1.5.0/go.mod h1:g4AWEaM3yOg3HYfnJ3YIawPnVdXJh9QME85blwSAmyw= github.com/OneOfOne/xxhash v1.2.2 h1:KMrpdQIwFcEqXDklaen+P1axHaj9BSKzvpUUfnHldSE= @@ -72,19 +39,11 @@ github.com/Zilliqa/gozilliqa-sdk v1.2.1-0.20201201074141-dd0ecada1be6/go.mod h1: github.com/aead/siphash v1.0.1/go.mod h1:Nywa3cDsYNNK3gaciGTWPwHt0wlpNV15vwmswBAUSII= github.com/ajstarks/svgo v0.0.0-20180226025133-644b8db467af/go.mod h1:K08gAheRH3/J6wwsYMMT4xOr94bZjxIelGM0+d/wbFw= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= -github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= -github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/allegro/bigcache v1.2.1-0.20190218064605-e24eb225f156/go.mod h1:Cb/ax3seSYIx7SuZdm2G2xzfwmv3TPSk2ucNfQESPXM= github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883/go.mod h1:rCTlJbsFo29Kk6CurOXKm700vrz8f0KW0JNfpkRJY/8= -github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= github.com/apache/arrow/go/arrow v0.0.0-20191024131854-af6fa24be0db/go.mod h1:VTxUBvSJ3s3eHAg65PNgrsn5BtqCRPdmyXh6rAfdxN0= -github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= -github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= -github.com/armon/go-metrics v0.3.10/go.mod h1:4O98XIr/9W0sxpJ8UaYkvjk10Iff7SnFrb4QAOwNTFc= -github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= -github.com/armon/go-radix v1.0.0/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= github.com/aws/aws-sdk-go-v2 v1.2.0/go.mod h1:zEQs02YRBw1DjK0PoJv3ygDYOFTre1ejlJWl8FwAuQo= github.com/aws/aws-sdk-go-v2/config v1.1.1/go.mod h1:0XsVy9lBI/BCXm+2Tuvt39YmdHwS5unDQmxZOYe8F5Y= github.com/aws/aws-sdk-go-v2/credentials v1.1.1/go.mod h1:mM2iIjwl7LULWtS6JCACyInboHirisUUdkBPoTHMOUo= @@ -98,8 +57,6 @@ github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLj github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= -github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= -github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= github.com/bmizerany/pat v0.0.0-20170815010413-6226ea591a40/go.mod h1:8rLXio+WjiTceGBHIoTvn60HIbs7Hm7bcHjyrSqYB9c= github.com/boltdb/bolt v1.3.1/go.mod h1:clJnj/oiGkjum5o1McbSZDSLxVThjynRyGBgiAx27Ps= github.com/btcsuite/btcd v0.0.0-20190315201642-aa6e0f35703c/go.mod h1:DrZx5ec/dmnfpw9KyYoQyYo7d0KEvTkk/5M/vbZjAr8= @@ -130,29 +87,15 @@ github.com/c-bata/go-prompt v0.2.2/go.mod h1:VzqtzE2ksDBcdln8G7mk2RX9QyGjH+OVqOC github.com/cenkalti/backoff v2.2.1+incompatible h1:tNowT99t7UNflLxfYYSlKYsBpXdEet03Pg2g16Swow4= github.com/cenkalti/backoff v2.2.1+incompatible/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= -github.com/census-instrumentation/opencensus-proto v0.3.0/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/cp v0.1.0/go.mod h1:SOGHArjBr4JWaSDEVpWpo/hNg6RoKrls6Oh40hiwW+s= github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= -github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= -github.com/circonus-labs/circonus-gometrics v2.3.1+incompatible/go.mod h1:nmEj6Dob7S7YxXgwXpfOuvO54S+tGdZdw9fuRZt25Ag= -github.com/circonus-labs/circonusllhist v0.1.3/go.mod h1:kMXHVDlOchFAehlya5ePtbp5jckzBHf4XRpQvBOLI+I= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cloudflare/cloudflare-go v0.14.0/go.mod h1:EnwdgGMaFOruiPZRFSgn+TsQ3hQ7C/YWzIGLeu5c304= -github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= -github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= -github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= -github.com/cncf/udpa/go v0.0.0-20210930031921-04548b0d99d4/go.mod h1:6pvJx4me5XPnfI9Z40ddWsdw2W/uZgQLFXToKeRcDiI= -github.com/cncf/xds/go v0.0.0-20210312221358-fbca930ec8ed/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= -github.com/cncf/xds/go v0.0.0-20210805033703-aa0b78936158/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= -github.com/cncf/xds/go v0.0.0-20210922020428-25de7278fc84/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= -github.com/cncf/xds/go v0.0.0-20211001041855-01bcc9b48dfe/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= -github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= -github.com/cncf/xds/go v0.0.0-20211130200136-a8f946100490/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/coinbase/kryptology v1.8.0 h1:Aoq4gdTsJhSU3lNWsD5BWmFSz2pE0GlmrljaOxepdYY= github.com/coinbase/kryptology v1.8.0/go.mod h1:RYXOAPdzOGUe3qlSFkMGn58i3xUA8hmxYHksuq+8ciI= github.com/coinbase/rosetta-sdk-go v0.7.9 h1:lqllBjMnazTjIqYrOGv8h8jxjg9+hJazIGZr9ZvoCcA= @@ -165,8 +108,6 @@ github.com/consensys/gnark-crypto v0.5.3/go.mod h1:hOdPlWQV1gDLp7faZVeg8Y0iEPFaO github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk= github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= -github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= -github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE= github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/cpuguy83/go-md2man/v2 v2.0.1/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= @@ -204,28 +145,17 @@ github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4 github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/eclipse/paho.mqtt.golang v1.2.0/go.mod h1:H9keYFcgq3Qr5OUJm/JZI/i6U7joQ8SYLhZwfeOo6Ts= github.com/edsrzf/mmap-go v1.0.0/go.mod h1:YO35OhQPt3KJa3ryjFM5Bs14WD66h8eGKpfaBNrHW5M= -github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= -github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= -github.com/envoyproxy/go-control-plane v0.9.7/go.mod h1:cwu0lG7PUMfa9snN8LXBig5ynNVH9qI8YYLbd1fK2po= -github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= -github.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= -github.com/envoyproxy/go-control-plane v0.9.9-0.20210512163311-63b5d3c536b0/go.mod h1:hliV/p42l8fGbc6Y9bQ70uLwIvmJyVE5k4iMKlh8wCQ= -github.com/envoyproxy/go-control-plane v0.9.10-0.20210907150352-cf90f659a021/go.mod h1:AFq3mo9L8Lqqiid3OhADV3RfLJnjiw63cSpi+fDTRC0= -github.com/envoyproxy/go-control-plane v0.10.1/go.mod h1:AY7fTTXNdv/aJ2O5jwpxAPOWUZ7hQAEvzN5Pf27BkQQ= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= -github.com/envoyproxy/protoc-gen-validate v0.6.2/go.mod h1:2t7qjJNvHPx8IjnBOzl9E9/baC+qXE/TeeyBRzgJDws= github.com/ethereum/go-ethereum v1.10.17 h1:XEcumY+qSr1cZQaWsQs5Kck3FHB0V2RiMHPdTBJ+oT8= github.com/ethereum/go-ethereum v1.10.17/go.mod h1:Lt5WzjM07XlXc95YzrhosmR4J9Ahd6X2wyEV2SvGhk0= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= -github.com/fatih/color v1.9.0/go.mod h1:eQcE1qtQxscV5RaZvpXrrb8Drkc3/DdQ+uUYCNjL+zU= github.com/fatih/color v1.13.0 h1:8LOYc1KYPPmyKMuN8QV2DNRWNbLo6LZ0iLs8+mlH53w= github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk= github.com/fjl/memsize v0.0.0-20190710130421-bcb5799ab5e5/go.mod h1:VvhXpOYNQvB+uIk2RvXzuaQtkQJzzIx6lSBe1xv7hi0= github.com/fogleman/gg v1.2.1-0.20190220221249-0403632d5b90/go.mod h1:R/bRT+9gY/C5z7JzPU0zXsXHKM4/ayA+zqcVNZzPa1k= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= -github.com/fsnotify/fsnotify v1.5.1/go.mod h1:T3375wBYaZdLLcVNkcVbzGHY7f1l/uK5T5Ai1i3InKU= github.com/gballet/go-libpcsclite v0.0.0-20190607065134-2772fd86a8ff/go.mod h1:x7DCsMOv1taUwEWCzT4cmDeAkigA5/QCwUodaVOe8Ww= github.com/getkin/kin-openapi v0.53.0/go.mod h1:7Yn5whZr5kJi6t+kShccXS8ae1APpYTW6yheSwk8Yi4= github.com/getkin/kin-openapi v0.61.0/go.mod h1:7Yn5whZr5kJi6t+kShccXS8ae1APpYTW6yheSwk8Yi4= @@ -235,9 +165,7 @@ github.com/glycerine/goconvey v0.0.0-20190410193231-58a59202ab31/go.mod h1:Ogl1T github.com/go-chi/chi/v5 v5.0.0/go.mod h1:BBug9lr0cqtdAhsu6R4AAdvufI0/XBzAQSsUqJpoZOs= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= -github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= -github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= github.com/go-ole/go-ole v1.2.1/go.mod h1:7FAglXiTm7HKlQRDeOQ6ZNUHidzCWXuZWq/1dTyBNF8= @@ -246,34 +174,21 @@ github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh github.com/go-sourcemap/sourcemap v2.1.3+incompatible/go.mod h1:F8jJfvm2KbVjc5NqelyYJmf/v5J0dwNLS2mL4sNA1Jg= github.com/go-sql-driver/mysql v1.4.1/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= -github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/gofrs/uuid v3.3.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= -github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/golang-jwt/jwt/v4 v4.3.0/go.mod h1:/xlHOz8bRuivTWchD4jCa+NbatV+wEUSzwAxVc6locg= github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0/go.mod h1:E/TSTwGwJL78qG/PmXZO1EjYhfJinVAhrmmHX6Z8B9k= github.com/golang/geo v0.0.0-20190916061304-5b978397cfec/go.mod h1:QZ0nwyI2jOfgRAoBvP+ab5aRr7c9x7lhGEJrKvBwjWI= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= -github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= -github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= -github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= -github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4= -github.com/golang/mock v1.5.0/go.mod h1:CWnOUgYIOo4TcNZ0wHX3YZCqsaM1I1Jvs6v3mP3KVu8= -github.com/golang/mock v1.6.0/go.mod h1:p6yTPP+5HYm5mzsMV8JkE6ZKdX+/wYM6Hr+LicevLPs= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= -github.com/golang/protobuf v1.3.4/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= -github.com/golang/protobuf v1.3.5/go.mod h1:6O5/vntMXwX2lRkT1hjjk0nAC1IDOTvTlVgjlRvqsdk= github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= @@ -283,7 +198,6 @@ github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QD github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= -github.com/golang/protobuf v1.5.1/go.mod h1:DopwsBzvsk0Fs44TXzsVbJyPhcCPeIwnvohx4u74HPM= github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw= github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= @@ -300,88 +214,37 @@ github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMyw github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.6 h1:BKbKCqvP6I+rmFHt06ZmyQtvB8xAkWdhFyr0ZUNZcxQ= github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/gofuzz v1.1.1-0.20200604201612-c04b05f3adfa/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= -github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= -github.com/google/martian/v3 v3.1.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= -github.com/google/martian/v3 v3.2.1/go.mod h1:oBOf6HBosgwRXnUGWUB05QECsc6uvmMiJ3+6W4l/CUk= github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/pprof v0.0.0-20201023163331-3e6fc7fc9c4c/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/pprof v0.0.0-20201203190320-1bf35d6f28c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/pprof v0.0.0-20210122040257-d980be63207e/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/pprof v0.0.0-20210226084205-cbba55b83ad5/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/pprof v0.0.0-20210601050228-01bbb1931b22/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/pprof v0.0.0-20210609004039-a478d1d731e9/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.2.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= -github.com/googleapis/gax-go/v2 v2.1.0/go.mod h1:Q3nei7sK6ybPYH7twZdmQpAd1MKb7pfu6SK+H1/DsU0= -github.com/googleapis/gax-go/v2 v2.1.1/go.mod h1:hddJymUZASv3XPyGkUpKj8pPO47Rmb0eJc8R6ouapiM= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= github.com/gorilla/websocket v1.4.1/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/graph-gophers/graphql-go v1.3.0/go.mod h1:9CQHMSxwO4MprSdzoIEobiHpoLtHm77vfxsvsIN5Vuc= -github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= github.com/gtank/merlin v0.1.1/go.mod h1:T86dnYJhcGOh5BjZFCJWTDeTK7XW8uE+E21Cy/bIQ+s= -github.com/hashicorp/consul/api v1.11.0/go.mod h1:XjsvQN+RJGWI2TWy1/kqaE16HrR2J/FWgkYjdZQsX9M= -github.com/hashicorp/consul/sdk v0.8.0/go.mod h1:GBvyrGALthsZObzUGsfgHZQDXjg4lOjagTIwIR1vPms= -github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= github.com/hashicorp/go-bexpr v0.1.10/go.mod h1:oxlubA2vC/gFVfX1A6JGp7ls7uCDlfJn732ehYYg+g0= -github.com/hashicorp/go-cleanhttp v0.5.0/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= -github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= -github.com/hashicorp/go-cleanhttp v0.5.2/go.mod h1:kO/YDlP8L1346E6Sodw+PrpBSV4/SoxCXGY6BqNFT48= -github.com/hashicorp/go-hclog v0.12.0/go.mod h1:whpDNt7SSdeAju8AWKIWsul05p54N/39EeqMAyrmvFQ= -github.com/hashicorp/go-hclog v1.0.0/go.mod h1:whpDNt7SSdeAju8AWKIWsul05p54N/39EeqMAyrmvFQ= -github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= -github.com/hashicorp/go-immutable-radix v1.3.1/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= -github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM= -github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk= -github.com/hashicorp/go-multierror v1.1.0/go.mod h1:spPvp8C1qA32ftKqdAHm4hHTbPw+vmowP0z+KUhOZdA= -github.com/hashicorp/go-retryablehttp v0.5.3/go.mod h1:9B5zBasrRhHXnJnui7y6sL7es7NDiJgTc6Er0maI1Xs= -github.com/hashicorp/go-rootcerts v1.0.2/go.mod h1:pqUvnprVnM5bf7AOirdbb01K4ccR319Vf4pU3K5EGc8= -github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU= -github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4= -github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= -github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= -github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= github.com/hashicorp/golang-lru v0.5.5-0.20210104140557-80c98217689d/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= -github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64= -github.com/hashicorp/mdns v1.0.1/go.mod h1:4gW7WsVCke5TE7EPeYliwHlRUyBtfCwuFwuMg2DmyNY= -github.com/hashicorp/mdns v1.0.4/go.mod h1:mtBihi+LeNXGtG8L9dX59gAEa12BDtBQSp4v/YAJqrc= -github.com/hashicorp/memberlist v0.2.2/go.mod h1:MS2lj3INKhZjWNqd3N0m3J+Jxf3DAOnAH9VT3Sh9MUE= -github.com/hashicorp/memberlist v0.3.0/go.mod h1:MS2lj3INKhZjWNqd3N0m3J+Jxf3DAOnAH9VT3Sh9MUE= -github.com/hashicorp/serf v0.9.5/go.mod h1:UWDWwZeL5cuWDJdl0C6wrvrUwEqtQ4ZKBKKENpqIUyk= -github.com/hashicorp/serf v0.9.6/go.mod h1:TXZNMjZQijwlDvp+r0b63xZ45H7JmCmgg4gpTwn9UV4= github.com/holiman/bloomfilter/v2 v2.0.3/go.mod h1:zpoh+gs7qcpqrHr3dB55AMiJwo0iURXE7ZOP9L9hSkA= github.com/holiman/uint256 v1.2.0/go.mod h1:y4ga/t+u+Xwd7CpDgZESaRcWy0I7XMlTMA25ApIH5Jw= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/huin/goupnp v1.0.3-0.20220313090229-ca81a64b4204/go.mod h1:ZxNlw5WqJj6wSsRK5+YfflQGXYfccj5VgQsMNixHM7Y= github.com/huin/goutil v0.0.0-20170803182201-1ca381bf3150/go.mod h1:PpLOETDnJ0o3iZrZfqZzyLl6l7F3c6L1oWn7OICBi6o= -github.com/iancoleman/strcase v0.2.0/go.mod h1:iwCmte+B7n89clKwxIoIXy/HfoL7AsD47ZCWhYzw7ho= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= -github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/influxdata/flux v0.65.1/go.mod h1:J754/zds0vvpfwuq7Gc2wRdVwEodfpCFM7mYlOw2LqY= @@ -403,9 +266,6 @@ github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHW github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= github.com/jrick/logrotate v1.0.0/go.mod h1:LNinyqDIJnpAur+b8yyulnQw/wDuN1+BYKlTRt3OuAQ= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= -github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= -github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= -github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= github.com/jsternberg/zap-logfmt v1.0.0/go.mod h1:uvPs/4X51zdkcm5jXl5SYoN+4RK21K8mysFmDaM/h+o= @@ -415,7 +275,6 @@ github.com/jung-kurt/gofpdf v1.0.3-0.20190309125859-24315acbbda5/go.mod h1:7Id9E github.com/jwilder/encoding v0.0.0-20170811194829-b4e1701a28ef/go.mod h1:Ct9fl0F6iIOGgxJ5npU/IUOhOhqlVrGjyIZc8/MagT0= github.com/karalabe/usb v0.0.2/go.mod h1:Od972xHfMJowv7NGVDiWVxk2zxnWgjLlJzE+F4F7AGU= github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= -github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/kkdai/bstream v0.0.0-20161212061736-f391b8402d23/go.mod h1:J+Gs4SYgM6CZQHDETBtE9HaSEkGmuNXF86RwHhHUvq4= github.com/klauspost/compress v1.4.0/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= @@ -425,10 +284,8 @@ github.com/klauspost/cpuid v0.0.0-20170728055534-ae7887de9fa5/go.mod h1:Pj4uuM52 github.com/klauspost/crc32 v0.0.0-20161016154125-cb6bfca970f6/go.mod h1:+ZoRqAPRLkC4NPOvfYeR5KNOrY6TD+/sAC3HXPZgDYg= github.com/klauspost/pgzip v1.0.2-0.20170402124221-0bf5dcad4ada/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= -github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= -github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.2.1 h1:Fmg33tUaq4/8ym9TJN1x7sLJnHVwhP33CNkpYV/7rwI= github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= @@ -443,27 +300,20 @@ github.com/leanovate/gopter v0.2.9/go.mod h1:U2L/78B+KVFIx2VmW6onHJQzXtFb+p5y3y2 github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/lucasjones/reggen v0.0.0-20180717132126-cdb49ff09d77 h1:6xiz3+ZczT3M4+I+JLpcPGG1bQKm8067HktB17EDWEE= github.com/lucasjones/reggen v0.0.0-20180717132126-cdb49ff09d77/go.mod h1:5ELEyG+X8f+meRWHuqUOewBOhvHkl7M76pdGEansxW4= -github.com/lyft/protoc-gen-star v0.5.3/go.mod h1:V0xaHgaf5oCCqmcxYcWiDfTiKsZsRc87/1qhoTACD8w= github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= -github.com/magiconair/properties v1.8.5/go.mod h1:y3VJvCyxH9uVvJTWEGAELF3aiYNyPKd5NZ3oSwXrF60= github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/matryer/moq v0.0.0-20190312154309-6cfb0558e1bd/go.mod h1:9ELz6aaclSIGnZBoaSLZ3NAl1VTufbOrXBPvtcy6WiQ= github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= -github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= -github.com/mattn/go-colorable v0.1.6/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= github.com/mattn/go-colorable v0.1.7/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= github.com/mattn/go-colorable v0.1.8/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= github.com/mattn/go-colorable v0.1.9/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= github.com/mattn/go-colorable v0.1.12 h1:jF+Du6AlPIjs2BiUiQlKOX0rt3SujHxPnksPKZbaA40= github.com/mattn/go-colorable v0.1.12/go.mod h1:u5H1YNBxpqRaxsYJYSkiCWKzEfiAb1Gb520KVy5xxl4= -github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.9/go.mod h1:YNRxwqDuOph6SZLI9vUUz6OYw3QyUt7WiY2yME+cCiQ= -github.com/mattn/go-isatty v0.0.10/go.mod h1:qgIWMr58cqv1PHHyhnkY9lrL7etaEgOFcMEpPG5Rm84= -github.com/mattn/go-isatty v0.0.11/go.mod h1:PhnuNfih5lzO57/f3n+odYbM4JtupLOxQOAqxQCu2WE= github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= github.com/mattn/go-isatty v0.0.14 h1:yVuAays6BHfxijgZPzw+3Zlu5yQgKGP2/hcQbHb7S9Y= github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= @@ -473,24 +323,15 @@ github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m github.com/mattn/go-sqlite3 v1.11.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= github.com/mattn/go-tty v0.0.0-20180907095812-13ff1204f104/go.mod h1:XPvLUNfbS4fJH25nqRHfWLMa1ONC8Amw+mIA639KxkE= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= -github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= -github.com/miekg/dns v1.1.26/go.mod h1:bPDLeHnStXmXAq1m/Ch/hvfNHr14JKNPMBo3VZKjuso= -github.com/miekg/dns v1.1.41/go.mod h1:p6aan82bvRIyn+zDIv9xYNUpwa73JcSh9BKwknJysuI= github.com/mimoo/StrobeGo v0.0.0-20181016162300-f8f6d4d2b643/go.mod h1:43+3pMjjKimDBf5Kr4ZFNGbLql1zKkbImw+fZbw3geM= -github.com/mitchellh/cli v1.1.0/go.mod h1:xcISNoH86gajksDmfB23e/pu+B+GeFRMYmoHXxx3xhI= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= -github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI= -github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/mitchellh/mapstructure v1.4.1/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/mitchellh/mapstructure v1.4.3 h1:OVowDSCllw/YjdLkam3/sm7wEtOy59d8ndGgCcyj8cs= github.com/mitchellh/mapstructure v1.4.3/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/mitchellh/pointerstructure v1.2.0/go.mod h1:BRAsLI5zgXmw97Lf6s25bs8ohIXc3tViBH44KcwB2g4= -github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= -github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= -github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= github.com/modocache/gover v0.0.0-20171022184752-b58185e213c5/go.mod h1:caMODM3PzxT8aQXRPkAt8xlV/e7d7w8GM5g0fa5F0D8= github.com/mschoch/smat v0.0.0-20160514031455-90eadee771ae/go.mod h1:qAyveg+e4CE+eKJXWVjKXM4ck2QobLqTDytGJbLLhJg= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= @@ -513,11 +354,8 @@ github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1y github.com/opentracing/opentracing-go v1.0.2/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= github.com/opentracing/opentracing-go v1.0.3-0.20180606204148-bd9c31933947/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= -github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= -github.com/pascaldekloe/goe v0.1.0/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= github.com/paulbellamy/ratecounter v0.2.0/go.mod h1:Hfx1hDpSGoqxkVVpBi/IlYD7kChlfo5C6hzIHwPqfFE= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= -github.com/pelletier/go-toml v1.9.4/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= github.com/peterh/liner v1.0.1-0.20180619022028-8c1271fcf47f/go.mod h1:xIteQHvHuaLYG9IFj6mSxM0fCKrs34IrEQUhOYuGPHc= github.com/peterh/liner v1.1.1-0.20190123174540-a2c9a5303de7/go.mod h1:CRroGNssyjTd/qIG2FyxByd2S8JEAZXBl4qUrZf8GS0= github.com/philhofer/fwd v1.0.0/go.mod h1:gk3iGcWd9+svBvR0sR+KPcfE+RNWozjowpeBVG3ZVNU= @@ -526,38 +364,27 @@ github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINE github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pkg/sftp v1.10.1/go.mod h1:lYOWFsE0bwd1+KfKJaKeuokY15vzFx25BLbzYYoAxZI= github.com/pkg/term v0.0.0-20180730021639-bffc007b7fd5/go.mod h1:eCbImbZ95eXtAUIbLAuAVnBnwf83mjf6QIVH8SHYwqQ= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI= -github.com/posener/complete v1.2.3/go.mod h1:WZIdtGGp+qx0sLrYKtIRAruyNpv6hFCicSgv7Sy7s/s= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= -github.com/prometheus/client_golang v1.4.0/go.mod h1:e9GMxYsXl05ICDXkRhurwBS4Q3OK1iX/F2sw+iXX5zU= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.6.0/go.mod h1:eBmuwkDJBwy6iBfxCBob6t6dR6ENT/y+J+Zk0j9GMYc= -github.com/prometheus/common v0.9.1/go.mod h1:yhUN8i9wzaXS3w1O07YhxHEBxD+W35wd8bs7vj7HSQ4= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= -github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A= github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= github.com/retailnext/hllpp v1.0.1-0.20180308014038-101a6d2f8b52/go.mod h1:RDpi1RftBQPUCDRw6SmxeaREsAaRKnOclghuzp/WRzc= github.com/rjeczalik/notify v0.9.1/go.mod h1:rKwnCoCGeuQnwBtTSPL9Dad03Vh2n40ePRrjvIXnJho= -github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rs/cors v1.7.0/go.mod h1:gFx+x8UowdsKA9AchylcLynDq+nNFfI8FkUZdN/jGCU= github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= -github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= -github.com/sagikazarmark/crypt v0.3.0/go.mod h1:uD/D+6UF4SrIR1uGEv7bBNkNqLGqUr43MRiaGWX1Nig= -github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= github.com/segmentio/fasthash v1.0.3 h1:EI9+KE1EwvMLBWwjpRDc+fEM+prwxDYbslddQGtrmhM= github.com/segmentio/fasthash v1.0.3/go.mod h1:waKX8l2N8yckOgmSsXJi7x1ZfdKZ4x7KRMzBtS3oedY= github.com/segmentio/kafka-go v0.1.0/go.mod h1:X6itGqS9L4jDletMsxZ7Dz+JFWxM6JHfPOCvTvk+EJo= @@ -566,28 +393,22 @@ github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAm github.com/shirou/gopsutil v3.21.4-0.20210419000835-c7a38de76ee5+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA= github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= -github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI= github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= -github.com/spf13/afero v1.3.3/go.mod h1:5KUK8ByomD5Ti5Artl0RtHeI5pTF7MIDuXL3yY520V4= -github.com/spf13/afero v1.6.0/go.mod h1:Ai8FlHk4v/PARR026UzYexafAt9roJ7LcLMAmO6Z93I= github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= -github.com/spf13/cast v1.4.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU= -github.com/spf13/cobra v1.3.0 h1:R7cSvGu+Vv+qX0gW5R/85dx2kmmJT5z5NM8ifdYjdn0= -github.com/spf13/cobra v1.3.0/go.mod h1:BrRVncBjOJa/eUcVVm9CE+oC6as8k+VYr4NY7WCi9V4= +github.com/spf13/cobra v1.4.0 h1:y+wJpx64xcgO1V+RcnwW0LEHxTKRi2ZDPSBjWnrg88Q= +github.com/spf13/cobra v1.4.0/go.mod h1:Wo4iy3BUC+X2Fybo0PDqwJIv3dNRiZLHQymsfxlB84g= github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= -github.com/spf13/jwalterweatherman v1.1.0/go.mod h1:aNWZUN0dPAAO/Ljvb5BEdw96iTZ0EXowPYD95IqWIGo= github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s= -github.com/spf13/viper v1.10.0/go.mod h1:SoyBPwAtKDzypXNDFKN5kzH7ppppbGZtls1UpIy5AsM= github.com/status-im/keycard-go v0.0.0-20190316090335-8537d3370df4/go.mod h1:RZLeN1LMWmRsyYjvAu+I6Dm9QmlDaIIt+Y+4Kd7Tp+Q= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1 h1:2vfRuCMp5sSVIDSqO8oNnWJq7mPa6KVP3iPIwFBuy8A= @@ -601,7 +422,6 @@ github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1 h1:5TQK59W5E3v0r2duFAb7P95B6hEeOyEnHRa8MjYSMTY= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7/go.mod h1:q4W45IWZaF22tdD+VEXcAWRA037jwmWEB5VWYORlTpc= github.com/tidwall/gjson v1.12.1/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= github.com/tidwall/gjson v1.14.0 h1:6aeJ0bzojgWLa82gDQHcx3S0Lr/O51I9bJ5nv6JFx5w= @@ -615,7 +435,6 @@ github.com/tidwall/sjson v1.2.4/go.mod h1:098SZ494YoMWPmMO6ct4dcFnqxwj9r/gF0Etp1 github.com/tinylib/msgp v1.0.2/go.mod h1:+d+yLhGm8mzTaHzB+wgMYrodPfmZrzkirds8fDWklFE= github.com/tklauser/go-sysconf v0.3.5/go.mod h1:MkWzOF4RMCshBAMXuhXJs64Rte09mITnppBXY/rYEFI= github.com/tklauser/numcpus v0.2.2/go.mod h1:x3qojaO3uyYt0i56EW/VUYs7uBvdl2fkfZFu0T9wgjM= -github.com/tv42/httpunix v0.0.0-20150427012821-b75d8614f926/go.mod h1:9ESjWnEqriFuLhtthL60Sar/7RFoluCcXsuvEwTV5KM= github.com/tyler-smith/go-bip39 v1.0.1-0.20181017060643-dbb3b84ba2ef/go.mod h1:sJ5fKU0s6JVwZjjcUEX2zFOnvq0ASQ2K9Zr6cf67kNs= github.com/tyler-smith/go-bip39 v1.0.2/go.mod h1:sJ5fKU0s6JVwZjjcUEX2zFOnvq0ASQ2K9Zr6cf67kNs= github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= @@ -631,22 +450,11 @@ github.com/willf/bitset v1.1.3/go.mod h1:RjeCKbqT1RxIR/KWY6phxZiaY1IyutSBfGjNPyS github.com/xlab/treeprint v0.0.0-20180616005107-d6fb6747feb6/go.mod h1:ce1O1j6UtZfjr22oyGxGLbauSBp2YVXpARAosm7dHBg= github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= github.com/ybbus/jsonrpc v2.1.2+incompatible/go.mod h1:XJrh1eMSzdIYFbM08flv0wp5G35eRniyeGut1z+LSiE= -github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= -go.etcd.io/etcd/api/v3 v3.5.1/go.mod h1:cbVKeC6lCfl7j/8jBhAK6aIYO9XOjdptoxU/nLQcPvs= -go.etcd.io/etcd/client/pkg/v3 v3.5.1/go.mod h1:IJHfcCEKxYu1Os13ZdwCwIUTUVGYTSAM3YSwc9/Ac1g= -go.etcd.io/etcd/client/v2 v2.305.1/go.mod h1:pMEacxZW7o8pg4CrFE7pquyCJJzZvkvdD2RibOCCCGs= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= -go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= -go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= -go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= -go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E= -go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.7.0 h1:ADUqmZGgLDDfbSL9ZmPxKTybcoEYHgpYfELNoN+7hsw= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= @@ -656,20 +464,16 @@ go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/ go.uber.org/multierr v1.6.0 h1:y6IPFStTAIT5Ytl7/XYmHvzXQ7S3g/IeZW9hyZ5thw4= go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= go.uber.org/zap v1.9.1/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= -go.uber.org/zap v1.17.0/go.mod h1:MXVU+bhUf/A7Xi2HNOnopQOrmycQ5Ih87HtOu4q5SSo= go.uber.org/zap v1.21.0 h1:WefMeulhovoZ2sYXz7st6K0sLj7bBhpiFaud4r4zST8= go.uber.org/zap v1.21.0/go.mod h1:wjWOCqI0f2ZZrJF/UufIOkiC8ii6tm1iqIsLo76RfJw= golang.org/x/crypto v0.0.0-20170930174604-9419663f5a44/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= -golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190426145343-a29dc8fdc734/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190909091759-094676da4a83/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20190923035154-9ee001bba392/go.mod h1:/lpIB1dKB+9EgE3H3cr1v9wB50oz8l4C4h62xy7jSTY= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200115085410-6d4e4cb37c7d/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200510223506-06a226fb4e37/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= @@ -690,9 +494,6 @@ golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm0 golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY= golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= -golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= -golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= -golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= golang.org/x/image v0.0.0-20180708004352-c73c2afc3b81/go.mod h1:ux5Hcp/YLpHSI86hEcLt0YII63i6oz57MZXIpbrjZUs= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= @@ -704,27 +505,17 @@ golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHl golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs= -golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= -golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= -golang.org/x/lint v0.0.0-20201208152925-83fdc39ff7b5/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= -golang.org/x/lint v0.0.0-20210508222113-6edffad5e616/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= -golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= -golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.5.0/go.mod h1:5OXOZSfqPIIbmVBIIKWRFfZjPR0E5r58TLhUjH0a2Ro= golang.org/x/net v0.0.0-20180719180050-a680a1efc54d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20181023162649-9b4f9f5ad519/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -735,40 +526,19 @@ golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20190923162816-aa69164e4478/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200222125558-5a598a2470a0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200501053045-e0ff5e5a1de5/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200506145744-7e3656a0809f/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200513185701-a91f0712d120/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= -golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20200813134508-3edf25e44fcc/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20201010224723-4f7140c49acb/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.0.0-20201031054903-ff519b6c9102/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.0.0-20201209123823-ac852fbbde11/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210220033124-5f55cee0dc0d/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= -golang.org/x/net v0.0.0-20210316092652-d523dce5a7f4/go.mod h1:RBQZq4jEuRlivfhVLdyRGr576XBO4/greRjx4P4O3yc= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= -golang.org/x/net v0.0.0-20210410081132-afb366fc7cd1/go.mod h1:9tjilg8BloeKEkVJvy7fQ90B1CfIiPueXVOjqfkSzI8= -golang.org/x/net v0.0.0-20210503060351-7fd8e65b6420/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210610132358-84b48f89b13b/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20210813160813-60bc85c4be6d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f h1:OfiFi4JbukWwe3lzw+xunroH1mnC1e2Gy5cxNJApiSY= golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= @@ -776,35 +546,19 @@ golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4Iltr golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.0.0-20200902213428-5d25da1a8d43/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20201109201403-9fd604954f58/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20201208152858-08078c50e5b5/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20210218202405-ba52d332ba99/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20210220000619-9bb904979d93/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20210313182246-cd4f82c27b84/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20210514164344-f6687ab2804c/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20210628180205-a41e5a781914/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20210805134026-6f1e6394065a/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20210819190943-2bc19b11175f/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20211005180243-6b3c2da341f1/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c h1:5KslGYwFpkhGh+Q16bwMP3cOontH8FOep7tGV86Y7SQ= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20181026203630-95b1ffbd15a5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -812,7 +566,6 @@ golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5h golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -821,71 +574,37 @@ golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190922100055-0a153f010e69/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190924154521-2837fb4f24fe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191008105621-543471e840be/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200107162124-548cf772de50/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200124204421-9fbb57f87de9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200331124033-c3d80250170d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200501052902-10377860bb8e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200511232937-7e40ca221e25/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200519105757-fe76b779f299/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200814200057-3d37ad5750ed/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200826173525-f9321e4c35a6/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200905004654-be1d3432aa8f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210104204734-6f8348627aad/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210220050731-9a76102bfb43/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210303074136-134d130e1a04/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210305230114-8fe3ee5dd75b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210315160823-c6e025ad8005/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210316164454-77fc1eacc6aa/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210324051608-47abb6519492/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210403161142-5e06dd20ab57/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210420205809-ac73e9fd8988/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210514084401-e8d321eab015/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210603125802-9665404d3644/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210806184541-e5e7981a1069/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210816183151-1e6c022a8912/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210823070655-63515b42dcdf/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210908233432-aa78b53d3365/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211019181941-9d821ace8654/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20211124211545-fe61309f8881/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211205182925-97ca703d548d h1:FjkYO/PPp4Wi0EAUOVLxePm7qVW4r4ctbWpURyuOD0E= golang.org/x/sys v0.0.0-20211205182925-97ca703d548d/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= -golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= @@ -896,7 +615,6 @@ golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20201208040808-7e3f01d25324/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180525024113-a5b4c53f6e8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -916,47 +634,16 @@ golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgw golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20190907020128-2ca718005c18/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191130070609-6e064ea0cf2d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191216173652-a0e659d51361/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200108203644-89082a384178/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200117161641-43d50277825c/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200122220014-bf1340f18c4a/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200204074204-1cc6d1ef6c74/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200212150539-ea181f53ac56/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200224181240-023911ca70b2/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200227222343-706bc42d1f0d/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= -golang.org/x/tools v0.0.0-20200312045724-11d5b4c81c7d/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= -golang.org/x/tools v0.0.0-20200331025713-a30bf2db82d4/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8= -golang.org/x/tools v0.0.0-20200501065659-ab2804fb9c9d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20200512131952-2bc93b1c0c88/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20200515010526-7d3b6ebf133d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20200618134242-20370b0cb4b2/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= -golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= -golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= -golang.org/x/tools v0.0.0-20200904185747-39188db58858/go.mod h1:Cj7w3i3Rnn0Xh82ur9kSqwfTHTeVxaDqrfMjpcNT6bE= -golang.org/x/tools v0.0.0-20201110124207-079ba7bd75cd/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.0.0-20201201161351-ac6f37ff4c2a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.0.0-20201208233053-a543418bbed2/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.0.0-20210105154028-b0ab187a4818/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= -golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= -golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= -golang.org/x/tools v0.1.3/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= -golang.org/x/tools v0.1.4/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -976,38 +663,11 @@ google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEn google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= google.golang.org/api v0.14.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= google.golang.org/api v0.15.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= -google.golang.org/api v0.17.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= -google.golang.org/api v0.18.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= -google.golang.org/api v0.19.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= -google.golang.org/api v0.20.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= -google.golang.org/api v0.22.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= -google.golang.org/api v0.24.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= -google.golang.org/api v0.28.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= -google.golang.org/api v0.29.0/go.mod h1:Lcubydp8VUV7KeIHD9z2Bys/sm/vGKnG1UHuDBSrHWM= -google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz5138Fc= -google.golang.org/api v0.35.0/go.mod h1:/XrVsuzM0rZmrsbjJutiuftIzeuTQcEeaYcSk/mQ1dg= -google.golang.org/api v0.36.0/go.mod h1:+z5ficQTmoYpPn8LCUNVpK5I7hwkpjbcgqA7I34qYtE= -google.golang.org/api v0.40.0/go.mod h1:fYKFpnQN0DsDSKRVRcQSDQNtqWPfM9i+zNPxepjRCQ8= -google.golang.org/api v0.41.0/go.mod h1:RkxM5lITDfTzmyKFPt+wGrCJbVfniCr2ool8kTBzRTU= -google.golang.org/api v0.43.0/go.mod h1:nQsDGjRXMo4lvh5hP0TKqF244gqhGcr/YSIykhUk/94= -google.golang.org/api v0.47.0/go.mod h1:Wbvgpq1HddcWVtzsVLyfLp8lDg6AA241LmgIL59tHXo= -google.golang.org/api v0.48.0/go.mod h1:71Pr1vy+TAZRPkPs/xlCf5SsU8WjuAWv1Pfjbtukyy4= -google.golang.org/api v0.50.0/go.mod h1:4bNT5pAuq5ji4SRZm+5QIkjny9JAyVD/3gaSihNefaw= -google.golang.org/api v0.51.0/go.mod h1:t4HdrdoNgyN5cbEfm7Lum0lcLDLiise1F8qDKX00sOU= -google.golang.org/api v0.54.0/go.mod h1:7C4bFFOvVDGXjfDTAsgGwDgAxRDeQ4X8NvUedIt6z3k= -google.golang.org/api v0.55.0/go.mod h1:38yMfeP1kfjsl8isn0tliTjIb1rJXcQi4UXlbqivdVE= -google.golang.org/api v0.56.0/go.mod h1:38yMfeP1kfjsl8isn0tliTjIb1rJXcQi4UXlbqivdVE= -google.golang.org/api v0.57.0/go.mod h1:dVPlbZyBo2/OjBpmvNdpn2GRm6rPy75jyU7bmhdrMgI= -google.golang.org/api v0.59.0/go.mod h1:sT2boj7M9YJxZzgeZqXogmhfmRWDtPzT31xkieUbuZU= -google.golang.org/api v0.61.0/go.mod h1:xQRti5UdCmoCEqFxcz93fTl338AVqDgyaDRuOZ3hg9I= -google.golang.org/api v0.62.0/go.mod h1:dKmwPCydfsad4qCH08MSdgWjfHOyfpd4VtDGgRFdavw= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= -google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= -google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= @@ -1022,85 +682,13 @@ google.golang.org/genproto v0.0.0-20191115194625-c23dd37a84c9/go.mod h1:n3cpQtvx google.golang.org/genproto v0.0.0-20191216164720-4f79533eabd1/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= google.golang.org/genproto v0.0.0-20191230161307-f3c370f40bfb/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= google.golang.org/genproto v0.0.0-20200108215221-bd8f9a0ef82f/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20200115191322-ca5a22157cba/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20200122232147-0452cf42e150/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20200204135345-fa8e72b47b90/go.mod h1:GmwEX6Z4W5gMy59cAlVYjN9JhxgbQH6Gn+gFDQe2lzA= -google.golang.org/genproto v0.0.0-20200212174721-66ed5ce911ce/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200224152610-e50cd9704f63/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200228133532-8c2c7df3a383/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200305110556-506484158171/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200312145019-da6875a35672/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200331122359-1ee6d9798940/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200430143042-b979b6f78d84/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200511104702-f5ebc3bea380/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200513103714-09dca8ec2884/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200515170657-fc4c6c6a6587/go.mod h1:YsZOwe1myG/8QRHRsmBRE1LrgQY60beZKjly0O1fX9U= google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= -google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7FcilCzHH/e9qn6dsT145K34l5v+OpcnNgKAAA= -google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20200904004341-0bd0a958aa1d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20201109203340-2640f1f9cdfb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20201201144952-b05cb90ed32e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20201210142538-e3217bee35cc/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20201214200347-8c77b98c765d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20210222152913-aa3ee6e6a81c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20210303154014-9728d6b83eeb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20210310155132-4ce2db91004e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20210319143718-93e7006c17a6/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20210402141018-6c239bbf2bb1/go.mod h1:9lPAdzaEmUacj36I+k7YKbEc5CXzPIeORRgDAUOu28A= -google.golang.org/genproto v0.0.0-20210513213006-bf773b8c8384/go.mod h1:P3QM42oQyzQSnHPnZ/vqoCdDmzH28fzWByN9asMeM8A= -google.golang.org/genproto v0.0.0-20210602131652-f16073e35f0c/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0= -google.golang.org/genproto v0.0.0-20210604141403-392c879c8b08/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0= -google.golang.org/genproto v0.0.0-20210608205507-b6d2f5bf0d7d/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0= -google.golang.org/genproto v0.0.0-20210624195500-8bfb893ecb84/go.mod h1:SzzZ/N+nwJDaO1kznhnlzqS8ocJICar6hYhVyhi++24= -google.golang.org/genproto v0.0.0-20210713002101-d411969a0d9a/go.mod h1:AxrInvYm1dci+enl5hChSFPOmmUF1+uAa/UsgNRWd7k= -google.golang.org/genproto v0.0.0-20210716133855-ce7ef5c701ea/go.mod h1:AxrInvYm1dci+enl5hChSFPOmmUF1+uAa/UsgNRWd7k= -google.golang.org/genproto v0.0.0-20210728212813-7823e685a01f/go.mod h1:ob2IJxKrgPT52GcgX759i1sleT07tiKowYBGbczaW48= -google.golang.org/genproto v0.0.0-20210805201207-89edb61ffb67/go.mod h1:ob2IJxKrgPT52GcgX759i1sleT07tiKowYBGbczaW48= -google.golang.org/genproto v0.0.0-20210813162853-db860fec028c/go.mod h1:cFeNkxwySK631ADgubI+/XFU/xp8FD5KIVV4rj8UC5w= -google.golang.org/genproto v0.0.0-20210821163610-241b8fcbd6c8/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= -google.golang.org/genproto v0.0.0-20210828152312-66f60bf46e71/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= -google.golang.org/genproto v0.0.0-20210831024726-fe130286e0e2/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= -google.golang.org/genproto v0.0.0-20210903162649-d08c68adba83/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= -google.golang.org/genproto v0.0.0-20210909211513-a8c4777a87af/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= -google.golang.org/genproto v0.0.0-20210924002016-3dee208752a0/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= -google.golang.org/genproto v0.0.0-20211008145708-270636b82663/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= -google.golang.org/genproto v0.0.0-20211028162531-8db9c33dc351/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= -google.golang.org/genproto v0.0.0-20211118181313-81c1377c94b1/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= -google.golang.org/genproto v0.0.0-20211129164237-f09f9a12af12/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= -google.golang.org/genproto v0.0.0-20211203200212-54befc351ae9/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= -google.golang.org/genproto v0.0.0-20211206160659-862468c7d6e0/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= -google.golang.org/genproto v0.0.0-20211208223120-3a66f561d7aa/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= -google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= -google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= -google.golang.org/grpc v1.28.0/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKal+60= -google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk= -google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= -google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= -google.golang.org/grpc v1.31.1/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= -google.golang.org/grpc v1.33.1/go.mod h1:fr5YgcSWrqhRRxogOsw7RzIpsmvOZ6IcH4kBYTpR3n0= -google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= -google.golang.org/grpc v1.34.0/go.mod h1:WotjhfgOW/POjDeRt8vscBtXq+2VjORFy659qA51WJ8= -google.golang.org/grpc v1.35.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= -google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= -google.golang.org/grpc v1.36.1/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= -google.golang.org/grpc v1.37.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= -google.golang.org/grpc v1.37.1/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= -google.golang.org/grpc v1.38.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= -google.golang.org/grpc v1.39.0/go.mod h1:PImNr+rS9TWYb2O4/emRugxiyHZ5JyHW5F+RPnDzfrE= -google.golang.org/grpc v1.39.1/go.mod h1:PImNr+rS9TWYb2O4/emRugxiyHZ5JyHW5F+RPnDzfrE= -google.golang.org/grpc v1.40.0/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34= -google.golang.org/grpc v1.40.1/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34= -google.golang.org/grpc v1.42.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU= -google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.1.0/go.mod h1:6Kw0yEErY5E/yWrBtf03jp27GLLJujG4z/JK95pnjjw= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= @@ -1109,7 +697,6 @@ google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzi google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= -google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4= google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= @@ -1123,7 +710,6 @@ gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntN gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= -gopkg.in/ini.v1 v1.66.2/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/natefinch/npipe.v2 v2.0.0-20160621034901-c1b8fa8bdcce/go.mod h1:5AcXVHNjg+BDxry382+8OKon8SEWiKktQR07RKPsv1c= gopkg.in/olebedev/go-duktape.v3 v3.0.0-20200619000410-60c24ae608a6/go.mod h1:uAJfkITjFhyEEuUfm7bsmCZRbW5WRq8s9EY8HZ6hCns= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= @@ -1132,7 +718,6 @@ gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= @@ -1146,11 +731,7 @@ honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWh honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= -honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= -honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= honnef.co/go/tools v0.1.3/go.mod h1:NgwopIslSNH47DimFoV78dnkksY2EFtX0ajyb3K/las= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4= -rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= -rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= rsc.io/tmplfunc v0.0.3/go.mod h1:AG3sTPzElb1Io3Yg4voV9AGZJuleGAwaVRxL9M49PhA= From 91be1afedf7f25b155e8c5ef5a4ddc78580b90f3 Mon Sep 17 00:00:00 2001 From: Shi Cheng Date: Wed, 8 Jun 2022 17:25:23 -0400 Subject: [PATCH 323/376] ROSE-320: prototype of check:spec --- cmd/check_spec.go | 128 ++++++++++++++++++++++++++++++++++++++++ cmd/check_spec_utils.go | 33 +++++++++++ cmd/root.go | 3 + 3 files changed, 164 insertions(+) create mode 100644 cmd/check_spec.go create mode 100644 cmd/check_spec_utils.go diff --git a/cmd/check_spec.go b/cmd/check_spec.go new file mode 100644 index 00000000..b0ee289e --- /dev/null +++ b/cmd/check_spec.go @@ -0,0 +1,128 @@ +// Copyright 2022 Coinbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package cmd + +import ( + "context" + "fmt" + "time" + + "github.com/coinbase/rosetta-cli/pkg/results" + "github.com/coinbase/rosetta-sdk-go/fetcher" + "github.com/spf13/cobra" +) + +var ( + checkSpecCmd = &cobra.Command{ + Use: "check:spec", + Short: "Check a Rosetta implementation satisfies Rosetta spec", + Long: `Detailed Rosetta spec can be found in https://www.rosetta-api.org/docs/Reference.html. + Specifically, check:spec will examine the response from all data and construction API endpoints, + and verifiy they have required fields and the values are properly populated and formatted.`, + RunE: runCheckSpecCmd, + } +) + +// checkSpec struct should implement following interface +// type checkSpecer interface { +// NetworkList() error +// NetworkOptions() error +// NetworkStatus(ctx context.Context) error + +// AccountBalance() error +// AccountCoins() error + +// Block() error +// BlockTransaction() error + +// ConstructionCombine() error +// ConstructionHash() error +// ConstructionMetadata() error +// ConstructionParse() error +// ConstructionPayloads() error +// ConstructionPreprocess() error +// ConstructionSubmit() error + +// Error() error +// MultipleModes() error +// } +type checkSpec struct { + fetcher *fetcher.Fetcher +} + +func newCheckSpec(ctx context.Context) (*checkSpec, error) { + fetcherOpts := []fetcher.Option{ + fetcher.WithMaxConnections(Config.MaxOnlineConnections), + fetcher.WithRetryElapsedTime(time.Duration(Config.RetryElapsedTime) * time.Second), + fetcher.WithTimeout(time.Duration(Config.HTTPTimeout) * time.Second), + fetcher.WithMaxRetries(Config.MaxRetries), + } + + if Config.ForceRetry { + fetcherOpts = append(fetcherOpts, fetcher.WithForceRetry()) + } + + fetcher := fetcher.New( + Config.OnlineURL, + fetcherOpts..., + ) + + _, _, fetchErr := fetcher.InitializeAsserter(ctx, Config.Network, Config.ValidationFile) + if fetchErr != nil { + return nil, results.ExitData( + Config, + nil, + nil, + fmt.Errorf("%w: unable to initialize asserter", fetchErr.Err), + "", + "", + ) + } + + return &checkSpec{ + fetcher: fetcher, + }, nil +} + +func (cs *checkSpec) NetworkStatus(ctx context.Context) error { + res, err := cs.fetcher.NetworkStatusRetry(ctx, Config.Network, nil) + if err != nil { + return fmt.Errorf("%w: unable to fetch network status", err.Err) + } + + if err := verifyBlockIdentifier(res.CurrentBlockIdentifier); err != nil { + return fmt.Errorf("%w", err) + } + + // TODO + return nil +} + +func runCheckSpecCmd(_ *cobra.Command, _ []string) error { + ctx := context.Background() + cs, err := newCheckSpec(ctx) + if err != nil { + return fmt.Errorf("%w: unable to create checkSpec object", err) + } + + if err = cs.NetworkStatus(ctx); err != nil { + return fmt.Errorf("%w: network status verification failed", err) + } + + // TODO: more checks + + fmt.Println("Successfully validated check:spec") + return nil +} diff --git a/cmd/check_spec_utils.go b/cmd/check_spec_utils.go new file mode 100644 index 00000000..0e8f4a63 --- /dev/null +++ b/cmd/check_spec_utils.go @@ -0,0 +1,33 @@ +// Copyright 2022 Coinbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package cmd + +import ( + "errors" + + "github.com/coinbase/rosetta-sdk-go/types" +) + +var ( + errBlockIdentifier = errors.New("BlockIdentifier must have both index and hash") +) + +func verifyBlockIdentifier(blockID *types.BlockIdentifier) error { + if blockID != nil && blockID.Hash != "" && blockID.Index >= 0 { + return nil + } + + return errBlockIdentifier +} diff --git a/cmd/root.go b/cmd/root.go index eda76cba..caf10515 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -306,6 +306,9 @@ default values.`, // Benchmark commands rootCmd.AddCommand(checkPerfCmd) + + // check:spec + rootCmd.AddCommand(checkSpecCmd) } func initConfig() { From cde3f6901ad8345d2c1c83046eaf5e0f794608b6 Mon Sep 17 00:00:00 2001 From: iris-zhang Date: Wed, 8 Jun 2022 20:29:57 -0700 Subject: [PATCH 324/376] Add check spec for network list --- cmd/check_spec.go | 71 ++++++++++++++++++++++++++++++++++++++++------- 1 file changed, 61 insertions(+), 10 deletions(-) diff --git a/cmd/check_spec.go b/cmd/check_spec.go index b0ee289e..9a8b1aea 100644 --- a/cmd/check_spec.go +++ b/cmd/check_spec.go @@ -39,7 +39,7 @@ var ( // type checkSpecer interface { // NetworkList() error // NetworkOptions() error -// NetworkStatus(ctx context.Context) error +// NetworkStatus(ctx context.Context) error // AccountBalance() error // AccountCoins() error @@ -59,45 +59,71 @@ var ( // MultipleModes() error // } type checkSpec struct { - fetcher *fetcher.Fetcher + onlineFetcher *fetcher.Fetcher + offlineFetcher *fetcher.Fetcher } func newCheckSpec(ctx context.Context) (*checkSpec, error) { - fetcherOpts := []fetcher.Option{ + onlineFetcherOpts := []fetcher.Option{ fetcher.WithMaxConnections(Config.MaxOnlineConnections), fetcher.WithRetryElapsedTime(time.Duration(Config.RetryElapsedTime) * time.Second), fetcher.WithTimeout(time.Duration(Config.HTTPTimeout) * time.Second), fetcher.WithMaxRetries(Config.MaxRetries), } + offlineFetcherOpts := []fetcher.Option{ + fetcher.WithMaxConnections(Config.Construction.MaxOfflineConnections), + fetcher.WithRetryElapsedTime(time.Duration(Config.RetryElapsedTime) * time.Second), + fetcher.WithTimeout(time.Duration(Config.HTTPTimeout) * time.Second), + fetcher.WithMaxRetries(Config.MaxRetries), + } + if Config.ForceRetry { - fetcherOpts = append(fetcherOpts, fetcher.WithForceRetry()) + onlineFetcherOpts = append(onlineFetcherOpts, fetcher.WithForceRetry()) + offlineFetcherOpts = append(offlineFetcherOpts, fetcher.WithForceRetry()) } - fetcher := fetcher.New( + onlineFetcher := fetcher.New( Config.OnlineURL, - fetcherOpts..., + onlineFetcherOpts..., + ) + offlineFetcher := fetcher.New( + Config.Construction.OfflineURL, + offlineFetcherOpts..., ) - _, _, fetchErr := fetcher.InitializeAsserter(ctx, Config.Network, Config.ValidationFile) + _, _, fetchErr := onlineFetcher.InitializeAsserter(ctx, Config.Network, Config.ValidationFile) + if fetchErr != nil { + return nil, results.ExitData( + Config, + nil, + nil, + fmt.Errorf("%w: unable to initialize asserter for online node fetcher", fetchErr.Err), + "", + "", + ) + } + + _, _, fetchErr = offlineFetcher.InitializeAsserter(ctx, Config.Network, Config.ValidationFile) if fetchErr != nil { return nil, results.ExitData( Config, nil, nil, - fmt.Errorf("%w: unable to initialize asserter", fetchErr.Err), + fmt.Errorf("%w: unable to initialize asserter for offline node fetcher", fetchErr.Err), "", "", ) } return &checkSpec{ - fetcher: fetcher, + onlineFetcher: onlineFetcher, + offlineFetcher: offlineFetcher, }, nil } func (cs *checkSpec) NetworkStatus(ctx context.Context) error { - res, err := cs.fetcher.NetworkStatusRetry(ctx, Config.Network, nil) + res, err := cs.onlineFetcher.NetworkStatusRetry(ctx, Config.Network, nil) if err != nil { return fmt.Errorf("%w: unable to fetch network status", err.Err) } @@ -110,6 +136,23 @@ func (cs *checkSpec) NetworkStatus(ctx context.Context) error { return nil } +func (cs *checkSpec) NetworkList(ctx context.Context, fetcher *fetcher.Fetcher) error { + networks, err := fetcher.NetworkList(ctx, nil) + if err != nil { + return fmt.Errorf("%w: unable to fetch network list", err.Err) + } + if len(networks.NetworkIdentifiers) == 0 { + return fmt.Errorf("network_identifiers are required") + } + for _, network := range networks.NetworkIdentifiers { + if network.Network == Config.Network.Network && + network.Blockchain == Config.Network.Blockchain { + return nil + } + } + return fmt.Errorf("network identifier in configuration file is not returned by /network/list") +} + func runCheckSpecCmd(_ *cobra.Command, _ []string) error { ctx := context.Background() cs, err := newCheckSpec(ctx) @@ -121,6 +164,14 @@ func runCheckSpecCmd(_ *cobra.Command, _ []string) error { return fmt.Errorf("%w: network status verification failed", err) } + if err = cs.NetworkList(ctx, cs.onlineFetcher); err != nil { + return fmt.Errorf("%w: online network list verification failed", err) + } + + if err = cs.NetworkList(ctx, cs.offlineFetcher); err != nil { + return fmt.Errorf("%w: offline network list verification failed", err) + } + // TODO: more checks fmt.Println("Successfully validated check:spec") From 96fa78ea4c2cdb0ded45b677436f261bdfc7505c Mon Sep 17 00:00:00 2001 From: Shi Cheng Date: Thu, 9 Jun 2022 15:21:26 -0400 Subject: [PATCH 325/376] ROSE-401: implement check:spec for data API endpoints --- cmd/check_spec.go | 84 ++++++++++++++++++++++++++++++--- cmd/check_spec_utils.go | 102 +++++++++++++++++++++++++++++++++++++++- 2 files changed, 177 insertions(+), 9 deletions(-) diff --git a/cmd/check_spec.go b/cmd/check_spec.go index b0ee289e..560a33a8 100644 --- a/cmd/check_spec.go +++ b/cmd/check_spec.go @@ -62,7 +62,7 @@ type checkSpec struct { fetcher *fetcher.Fetcher } -func newCheckSpec(ctx context.Context) (*checkSpec, error) { +func newCheckSpec(ctx context.Context, serverAddr string) (*checkSpec, error) { fetcherOpts := []fetcher.Option{ fetcher.WithMaxConnections(Config.MaxOnlineConnections), fetcher.WithRetryElapsedTime(time.Duration(Config.RetryElapsedTime) * time.Second), @@ -75,7 +75,7 @@ func newCheckSpec(ctx context.Context) (*checkSpec, error) { } fetcher := fetcher.New( - Config.OnlineURL, + serverAddr, fetcherOpts..., ) @@ -96,32 +96,102 @@ func newCheckSpec(ctx context.Context) (*checkSpec, error) { }, nil } +func (cs *checkSpec) NetworkOptions(ctx context.Context) error { + res, err := cs.fetcher.NetworkOptionsRetry(ctx, Config.Network, nil) + if err != nil { + return fmt.Errorf("%w: unable to fetch network options", err.Err) + } + + // version is required + if res.Version == nil { + return fmt.Errorf("%w: unable to find version in network/options response", errVersion) + } + + if err := validateVersion(res.Version.RosettaVersion); err != nil { + return fmt.Errorf("%w", err) + } + + if err := validateVersion(res.Version.NodeVersion); err != nil { + return fmt.Errorf("%w", err) + } + + // allow is required + if res.Allow == nil { + return fmt.Errorf("%w: unable to find allow in network/options response", errAllowNullPointer) + } + + if err := validateOperationStatuses(res.Allow.OperationStatuses); err != nil { + return fmt.Errorf("%w", err) + } + + if err := validateOperationTypes(res.Allow.OperationTypes); err != nil { + return fmt.Errorf("%w", err) + } + + if err := validateErrors(res.Allow.Errors); err != nil { + return fmt.Errorf("%w", err) + } + + if err := validateCallMethods(res.Allow.CallMethods); err != nil { + return fmt.Errorf("%w", err) + } + + if err := validateBalanceExemptions(res.Allow.BalanceExemptions); err != nil { + return fmt.Errorf("%w", err) + } + + return nil +} + func (cs *checkSpec) NetworkStatus(ctx context.Context) error { res, err := cs.fetcher.NetworkStatusRetry(ctx, Config.Network, nil) if err != nil { return fmt.Errorf("%w: unable to fetch network status", err.Err) } - if err := verifyBlockIdentifier(res.CurrentBlockIdentifier); err != nil { + // current_block_identifier is required + if err := validateBlockIdentifier(res.CurrentBlockIdentifier); err != nil { + return fmt.Errorf("%w", err) + } + + // current_block_timestamp is required + if err := validateTimestamp(res.CurrentBlockTimestamp); err != nil { + return fmt.Errorf("%w", err) + } + + // genesis_block_identifier is required + if err := validateBlockIdentifier(res.GenesisBlockIdentifier); err != nil { + return fmt.Errorf("%w", err) + } + + // peers is required + if err := validatePeers(res.Peers); err != nil { return fmt.Errorf("%w", err) } - // TODO return nil } func runCheckSpecCmd(_ *cobra.Command, _ []string) error { ctx := context.Background() - cs, err := newCheckSpec(ctx) + onlineCheckSpec, err := newCheckSpec(ctx, Config.OnlineURL) if err != nil { - return fmt.Errorf("%w: unable to create checkSpec object", err) + return fmt.Errorf("%w: unable to create checkSpec object with online URL", err) } - if err = cs.NetworkStatus(ctx); err != nil { + if err = onlineCheckSpec.NetworkStatus(ctx); err != nil { return fmt.Errorf("%w: network status verification failed", err) } // TODO: more checks + offlineCheckSpec, err := newCheckSpec(ctx, Config.Construction.OfflineURL) + if err != nil { + return fmt.Errorf("%w: unable to create checkSpec object with offline URL", err) + } + + if err = offlineCheckSpec.NetworkOptions(ctx); err != nil { + return fmt.Errorf("%w: network options verification failed", err) + } fmt.Println("Successfully validated check:spec") return nil diff --git a/cmd/check_spec_utils.go b/cmd/check_spec_utils.go index 0e8f4a63..c00cfc44 100644 --- a/cmd/check_spec_utils.go +++ b/cmd/check_spec_utils.go @@ -21,13 +21,111 @@ import ( ) var ( - errBlockIdentifier = errors.New("BlockIdentifier must have both index and hash") + errBlockIdentifier = errors.New("BlockIdentifier must have both index and hash") + errTimestamp = errors.New("Timestamp must be greater than or equal to 0") + errPeer = errors.New("Peer should be present and peer id should not be empty") + errVersion = errors.New("Version should not be empty") + errOperationStatus = errors.New("Operation status should not be empty") + errOperationTypeEmpty = errors.New("Operation type should not be empty") + + errErrorEmpty = errors.New("Error object can't be empty") + errErrorEmptyMessage = errors.New("Error object can't have empty message") + errErrorNegativeCode = errors.New("Error object can't have negative code") + + errCallMethodEmptyName = errors.New("Call method name can't be empty") + errBalanceExemptionEmpty = errors.New("Balance exemption can't be empty") + + errAllowNullPointer = errors.New("Null pointer to Allow object") ) -func verifyBlockIdentifier(blockID *types.BlockIdentifier) error { +func validateBlockIdentifier(blockID *types.BlockIdentifier) error { if blockID != nil && blockID.Hash != "" && blockID.Index >= 0 { return nil } return errBlockIdentifier } + +func validateTimestamp(time int64) error { + if time >= 0 { + return nil + } + + return errTimestamp +} + +func validatePeers(peers []*types.Peer) error { + for _, p := range peers { + if p == nil || p.PeerID == "" { + return errPeer + } + } + + return nil +} + +func validateVersion(version string) error { + if version == "" { + return errVersion + } + + return nil +} + +func validateOperationStatuses(oss []*types.OperationStatus) error { + for _, os := range oss { + if os == nil || os.Status == "" { + return errOperationStatus + } + } + + return nil +} + +func validateOperationTypes(ots []string) error { + for _, ot := range ots { + if ot == "" { + return errOperationTypeEmpty + } + } + + return nil +} + +func validateErrors(errors []*types.Error) error { + for _, err := range errors { + if err == nil { + return errErrorEmpty + } + + if err.Code < 0 { + return errErrorNegativeCode + } + + if err.Message == "" { + return errErrorEmptyMessage + } + } + + return nil +} + +func validateCallMethods(methods []string) error { + for _, m := range methods { + if m == "" { + return errCallMethodEmptyName + } + } + + return nil +} + +func validateBalanceExemptions(exs []*types.BalanceExemption) error { + for _, e := range exs { + if e == nil { + return errBalanceExemptionEmpty + } + } + + return nil +} From ee3dad34ab258ef047b55f92e020c6cc5ec03032 Mon Sep 17 00:00:00 2001 From: Shi Cheng Date: Tue, 14 Jun 2022 12:22:52 -0400 Subject: [PATCH 326/376] ROSE-404: Output test result in a readable format --- cmd/check_spec.go | 449 ++++++++++++++++++++++++++++++++-------- cmd/check_spec_utils.go | 273 +++++++++++++++++++++--- cmd/utils_shared.go | 31 +++ 3 files changed, 643 insertions(+), 110 deletions(-) create mode 100644 cmd/utils_shared.go diff --git a/cmd/check_spec.go b/cmd/check_spec.go index 457267dd..fbd04983 100644 --- a/cmd/check_spec.go +++ b/cmd/check_spec.go @@ -21,6 +21,7 @@ import ( "github.com/coinbase/rosetta-cli/pkg/results" "github.com/coinbase/rosetta-sdk-go/fetcher" + "github.com/coinbase/rosetta-sdk-go/types" "github.com/spf13/cobra" ) @@ -35,35 +36,16 @@ var ( } ) -// checkSpec struct should implement following interface -// type checkSpecer interface { -// NetworkList() error -// NetworkOptions() error -// NetworkStatus(ctx context.Context) error - -// AccountBalance() error -// AccountCoins() error - -// Block() error -// BlockTransaction() error - -// ConstructionCombine() error -// ConstructionHash() error -// ConstructionMetadata() error -// ConstructionParse() error -// ConstructionPayloads() error -// ConstructionPreprocess() error -// ConstructionSubmit() error - -// Error() error -// MultipleModes() error -// } type checkSpec struct { onlineFetcher *fetcher.Fetcher offlineFetcher *fetcher.Fetcher } func newCheckSpec(ctx context.Context) (*checkSpec, error) { + if Config.Construction == nil { + return nil, fmt.Errorf("%v", errRosettaConfigNoConstruction) + } + onlineFetcherOpts := []fetcher.Option{ fetcher.WithMaxConnections(Config.MaxOnlineConnections), fetcher.WithRetryElapsedTime(time.Duration(Config.RetryElapsedTime) * time.Second), @@ -98,19 +80,7 @@ func newCheckSpec(ctx context.Context) (*checkSpec, error) { Config, nil, nil, - fmt.Errorf("%w: unable to initialize asserter for online node fetcher", fetchErr.Err), - "", - "", - ) - } - - _, _, fetchErr = offlineFetcher.InitializeAsserter(ctx, Config.Network, Config.ValidationFile) - if fetchErr != nil { - return nil, results.ExitData( - Config, - nil, - nil, - fmt.Errorf("%w: unable to initialize asserter for offline node fetcher", fetchErr.Err), + fmt.Errorf("%v: unable to initialize asserter for online node fetcher", fetchErr.Err), "", "", ) @@ -122,127 +92,442 @@ func newCheckSpec(ctx context.Context) (*checkSpec, error) { }, nil } -func (cs *checkSpec) NetworkOptions(ctx context.Context) error { +func (cs *checkSpec) networkOptions(ctx context.Context) checkSpecOutput { + printInfo("validating /network/options ...\n") + output := checkSpecOutput{ + api: networkOptions, + validation: map[checkSpecRequirement]checkSpecStatus{ + version: checkSpecSuccess, + allow: checkSpecSuccess, + offlineMode: checkSpecSuccess, + }, + } + defer printInfo("/network/options validated\n") + res, err := cs.offlineFetcher.NetworkOptionsRetry(ctx, Config.Network, nil) if err != nil { - return fmt.Errorf("%w: unable to fetch network options", err.Err) + printError("%v: unable to fetch network options\n", err.Err) + markAllValidationsFailed(output) + return output } // version is required if res.Version == nil { - return fmt.Errorf("%w: unable to find version in network/options response", errVersion) + setValidationStatus(output, version, checkSpecFailure) + printError("%v: unable to find version in /network/options response\n", errVersionNullPointer) } if err := validateVersion(res.Version.RosettaVersion); err != nil { - return fmt.Errorf("%w", err) + setValidationStatus(output, version, checkSpecFailure) + printError("%v\n", err) } if err := validateVersion(res.Version.NodeVersion); err != nil { - return fmt.Errorf("%w", err) + setValidationStatus(output, version, checkSpecFailure) + printError("%v\n", err) } // allow is required if res.Allow == nil { - return fmt.Errorf("%w: unable to find allow in network/options response", errAllowNullPointer) + setValidationStatus(output, allow, checkSpecFailure) + printError("%v: unable to find allow in /network/options response\n", errAllowNullPointer) } if err := validateOperationStatuses(res.Allow.OperationStatuses); err != nil { - return fmt.Errorf("%w", err) + setValidationStatus(output, allow, checkSpecFailure) + printError("%v\n", err) } if err := validateOperationTypes(res.Allow.OperationTypes); err != nil { - return fmt.Errorf("%w", err) + setValidationStatus(output, allow, checkSpecFailure) + printError("%v\n", err) } if err := validateErrors(res.Allow.Errors); err != nil { - return fmt.Errorf("%w", err) + setValidationStatus(output, allow, checkSpecFailure) + printError("%v\n", err) } if err := validateCallMethods(res.Allow.CallMethods); err != nil { - return fmt.Errorf("%w", err) + setValidationStatus(output, allow, checkSpecFailure) + printError("%v\n", err) } if err := validateBalanceExemptions(res.Allow.BalanceExemptions); err != nil { - return fmt.Errorf("%w", err) + setValidationStatus(output, allow, checkSpecFailure) + printError("%v\n", err) } - return nil + return output } -func (cs *checkSpec) NetworkStatus(ctx context.Context) error { +func (cs *checkSpec) networkStatus(ctx context.Context) checkSpecOutput { + printInfo("validating /network/status ...\n") + output := checkSpecOutput{ + api: networkStatus, + validation: map[checkSpecRequirement]checkSpecStatus{ + currentBlockID: checkSpecSuccess, + currentBlockTime: checkSpecSuccess, + genesisBlockID: checkSpecSuccess, + }, + } + defer printInfo("/network/status validated\n") + res, err := cs.onlineFetcher.NetworkStatusRetry(ctx, Config.Network, nil) if err != nil { - return fmt.Errorf("%w: unable to fetch network status", err.Err) + printError("%v: unable to fetch network status\n", err.Err) + markAllValidationsFailed(output) + return output } // current_block_identifier is required if err := validateBlockIdentifier(res.CurrentBlockIdentifier); err != nil { - return fmt.Errorf("%w", err) + printError("%v\n", err) + setValidationStatus(output, currentBlockID, checkSpecFailure) } // current_block_timestamp is required if err := validateTimestamp(res.CurrentBlockTimestamp); err != nil { - return fmt.Errorf("%w", err) + printError("%v\n", err) + setValidationStatus(output, currentBlockTime, checkSpecFailure) } // genesis_block_identifier is required if err := validateBlockIdentifier(res.GenesisBlockIdentifier); err != nil { - return fmt.Errorf("%w", err) - } - - // peers is required - if err := validatePeers(res.Peers); err != nil { - return fmt.Errorf("%w", err) + printError("%v\n", err) + setValidationStatus(output, genesisBlockID, checkSpecFailure) } - return nil + return output } -func (cs *checkSpec) NetworkList(ctx context.Context, fetcher *fetcher.Fetcher) error { - networks, err := fetcher.NetworkList(ctx, nil) +func (cs *checkSpec) networkList(ctx context.Context) checkSpecOutput { + printInfo("validating /network/list ...\n") + output := checkSpecOutput{ + api: networkList, + validation: map[checkSpecRequirement]checkSpecStatus{ + networkIDs: checkSpecSuccess, + offlineMode: checkSpecSuccess, + staticNetworkID: checkSpecSuccess, + }, + } + defer printInfo("/network/list validated\n") + + networks, err := cs.offlineFetcher.NetworkList(ctx, nil) if err != nil { - return fmt.Errorf("%w: unable to fetch network list", err.Err) + printError("%v: unable to fetch network list", err.Err) + markAllValidationsFailed(output) + return output } + if len(networks.NetworkIdentifiers) == 0 { - return fmt.Errorf("network_identifiers are required") + printError("network_identifiers is required") + setValidationStatus(output, networkIDs, checkSpecFailure) } + for _, network := range networks.NetworkIdentifiers { - if network.Network == Config.Network.Network && - network.Blockchain == Config.Network.Blockchain { - return nil + if isEqual(network.Network, Config.Network.Network) && + isEqual(network.Blockchain, Config.Network.Blockchain) { + return output } } - return fmt.Errorf("network identifier in configuration file is not returned by /network/list") + + printError("network_identifier in configuration file is not returned by /network/list") + setValidationStatus(output, staticNetworkID, checkSpecFailure) + return output } -func runCheckSpecCmd(_ *cobra.Command, _ []string) error { - ctx := context.Background() - cs, err := newCheckSpec(ctx) +func (cs *checkSpec) accountBalance(ctx context.Context) checkSpecOutput { + printInfo("validating /account/balance ...\n") + output := checkSpecOutput{ + api: accountBalance, + validation: map[checkSpecRequirement]checkSpecStatus{ + blockID: checkSpecSuccess, + balances: checkSpecSuccess, + }, + } + defer printInfo("/account/balance validated\n") + + acct, partBlockID, currencies, err := cs.getAccount(ctx) if err != nil { - return fmt.Errorf("%w: unable to create checkSpec object with online URL", err) + markAllValidationsFailed(output) + printError("%v: unable to get an account\n", err) + return output + } + if acct == nil { + markAllValidationsFailed(output) + printError("%v\n", errAccountNullPointer) + return output + } + + // fetch account balance + block, amt, _, fetchErr := cs.onlineFetcher.AccountBalanceRetry( + ctx, + Config.Network, + acct, + partBlockID, + currencies) + if err != nil { + markAllValidationsFailed(output) + printError("%v: unable to fetch balance for account: %v\n", fetchErr.Err, *acct) + return output } - if err = cs.NetworkStatus(ctx); err != nil { - return fmt.Errorf("%w: network status verification failed", err) + // block_identifier is required + if err := validateBlockIdentifier(block); err != nil { + printError("%v\n", err) + setValidationStatus(output, blockID, checkSpecFailure) } - if err = cs.NetworkList(ctx, cs.onlineFetcher); err != nil { - return fmt.Errorf("%w: online network list verification failed", err) + // balances is required + if err := validateBalances(amt); err != nil { + printError("%v\n", err) + setValidationStatus(output, balances, checkSpecFailure) } - if err = cs.NetworkList(ctx, cs.offlineFetcher); err != nil { - return fmt.Errorf("%w: offline network list verification failed", err) + return output +} + +func (cs *checkSpec) accountCoins(ctx context.Context) checkSpecOutput { + printInfo("validating /account/coins ...\n") + output := checkSpecOutput{ + api: accountCoins, + validation: map[checkSpecRequirement]checkSpecStatus{ + blockID: checkSpecSuccess, + coins: checkSpecSuccess, + }, } + defer printInfo("/account/coins validated\n") + + if isUTXO() { + acct, _, currencies, err := cs.getAccount(ctx) + if err != nil { + printError("%v: unable to get an account\n", err) + markAllValidationsFailed(output) + return output + } + if err != nil { + printError("%v\n", errAccountNullPointer) + markAllValidationsFailed(output) + return output + } + + block, cs, _, fetchErr := cs.onlineFetcher.AccountCoinsRetry( + ctx, + Config.Network, + acct, + false, + currencies) + if fetchErr != nil { + printError("%v: unable to get coins for account: %v\n", fetchErr.Err, *acct) + markAllValidationsFailed(output) + return output + } + + // block_identifier is required + err = validateBlockIdentifier(block) + if err != nil { + printError("%v\n", err) + setValidationStatus(output, blockID, checkSpecFailure) + } + + // coins is required + err = validateCoins(cs) + if err != nil { + printError("%v\n", err) + setValidationStatus(output, coins, checkSpecFailure) + } + } + + return output +} - // TODO: more checks +func (cs *checkSpec) block(ctx context.Context) checkSpecOutput { + printInfo("validating /block ...\n") + output := checkSpecOutput{ + api: block, + validation: map[checkSpecRequirement]checkSpecStatus{ + idempotent: checkSpecSuccess, + defaultTip: checkSpecSuccess, + }, + } + defer printInfo("/block validated\n") + + res, fetchErr := cs.onlineFetcher.NetworkStatusRetry(ctx, Config.Network, nil) + if fetchErr != nil { + printError("%v: unable to get network status\n", fetchErr.Err) + markAllValidationsFailed(output) + return output + } + + // multiple calls with the same hash should return the same block + var block *types.Block + tip := res.CurrentBlockIdentifier + callTimes := 3 + + for i := 0; i < callTimes; i++ { + blockID := types.PartialBlockIdentifier{ + Hash: &tip.Hash, + } + b, fetchErr := cs.onlineFetcher.BlockRetry(ctx, Config.Network, &blockID) + if fetchErr != nil { + printError("%v: unable to fetch block %v\n", fetchErr.Err, blockID) + markAllValidationsFailed(output) + return output + } + + if block == nil { + block = b + } else if !isEqual(types.Hash(*block), types.Hash(*b)) { + printError("%v\n", errBlockNotIdempotent) + setValidationStatus(output, idempotent, checkSpecFailure) + } + } + + // fetch the tip block again + res, fetchErr = cs.onlineFetcher.NetworkStatusRetry(ctx, Config.Network, nil) + if fetchErr != nil { + printError("%v: unable to get network status\n", fetchErr.Err) + setValidationStatus(output, defaultTip, checkSpecFailure) + return output + } + tip = res.CurrentBlockIdentifier + + // tip shoud be returned if block_identifier is not specified + emptyBlockID := &types.PartialBlockIdentifier{} + block, fetchErr = cs.onlineFetcher.BlockRetry(ctx, Config.Network, emptyBlockID) + if fetchErr != nil { + printError("%v: unable to fetch tip block\n", fetchErr.Err) + setValidationStatus(output, defaultTip, checkSpecFailure) + return output + } + + // block index returned from /block should be >= the index returned by /network/status + if isNegative(block.BlockIdentifier.Index - tip.Index) { + printError("%v\n", errBlockTip) + setValidationStatus(output, defaultTip, checkSpecFailure) + } + + return output +} + +func (cs *checkSpec) errorObject(ctx context.Context) checkSpecOutput { + printInfo("validating error object ...\n") + output := checkSpecOutput{ + api: errorObject, + validation: map[checkSpecRequirement]checkSpecStatus{ + errorCode: checkSpecSuccess, + errorMessage: checkSpecSuccess, + }, + } + defer printInfo("error object validated\n") + + printInfo("%v\n", "sending request to /network/status ...") + emptyNetwork := &types.NetworkIdentifier{} + _, err := cs.onlineFetcher.NetworkStatusRetry(ctx, emptyNetwork, nil) + validateErrorObject(err, output) + + printInfo("%v\n", "sending request to /network/options ...") + _, err = cs.onlineFetcher.NetworkOptionsRetry(ctx, emptyNetwork, nil) + validateErrorObject(err, output) + + printInfo("%v\n", "sending request to /account/balance ...") + emptyAcct := &types.AccountIdentifier{} + emptyPartBlock := &types.PartialBlockIdentifier{} + emptyCur := []*types.Currency{} + _, _, _, err = cs.onlineFetcher.AccountBalanceRetry(ctx, emptyNetwork, emptyAcct, emptyPartBlock, emptyCur) + validateErrorObject(err, output) + + if isUTXO() { + printInfo("%v\n", "sending request to /account/coins ...") + _, _, _, err = cs.onlineFetcher.AccountCoinsRetry(ctx, emptyNetwork, emptyAcct, false, emptyCur) + validateErrorObject(err, output) + } else { + printInfo("%v\n", "skip /account/coins for account based chain") + } + + printInfo("%v\n", "sending request to /block ...") + _, err = cs.onlineFetcher.BlockRetry(ctx, emptyNetwork, emptyPartBlock) + validateErrorObject(err, output) + + printInfo("%v\n", "sending request to /block/transaction ...") + emptyTx := []*types.TransactionIdentifier{} + emptyBlock := &types.BlockIdentifier{} + _, err = cs.onlineFetcher.UnsafeTransactions(ctx, emptyNetwork, emptyBlock, emptyTx) + validateErrorObject(err, output) + + return output +} + +// Searching for an account backwards from the tip +func (cs *checkSpec) getAccount(ctx context.Context) ( + *types.AccountIdentifier, + *types.PartialBlockIdentifier, + []*types.Currency, + error) { + res, err := cs.onlineFetcher.NetworkStatusRetry(ctx, Config.Network, nil) + if err != nil { + return nil, nil, nil, fmt.Errorf("%v: unable to get network status", err.Err) + } + + var acct *types.AccountIdentifier + var blockID *types.PartialBlockIdentifier + tip := res.CurrentBlockIdentifier.Index + genesis := res.GenesisBlockIdentifier.Index + currencies := []*types.Currency{} + + for i := tip; i >= genesis && acct == nil; i-- { + blockID = &types.PartialBlockIdentifier{ + Index: &i, + } + + block, err := cs.onlineFetcher.BlockRetry(ctx, Config.Network, blockID) + if err != nil { + return nil, nil, nil, fmt.Errorf("%v: unable to fetch block at index: %v", err.Err, i) + } + + // looking for an account in block transactions + for _, tx := range block.Transactions { + for _, op := range tx.Operations { + if op.Account != nil && op.Amount.Currency != nil { + acct = op.Account + currencies = append(currencies, op.Amount.Currency) + break + } + } + + if acct != nil { + break + } + } + } + + return acct, blockID, currencies, nil +} + +func runCheckSpecCmd(_ *cobra.Command, _ []string) error { + ctx := context.Background() + cs, err := newCheckSpec(ctx) if err != nil { - return fmt.Errorf("%w: unable to create checkSpec object with offline URL", err) + return fmt.Errorf("%v: unable to create checkSpec object with online URL", err) } - if err = cs.NetworkOptions(ctx); err != nil { - return fmt.Errorf("%w: network options verification failed", err) + output := []checkSpecOutput{} + // validate api endpoints + output = append(output, cs.networkStatus(ctx)) + output = append(output, cs.networkList(ctx)) + output = append(output, cs.networkOptions(ctx)) + output = append(output, cs.accountBalance(ctx)) + output = append(output, cs.accountCoins(ctx)) + output = append(output, cs.block(ctx)) + output = append(output, cs.errorObject(ctx)) + output = append(output, twoModes()) + + printInfo("check:spec is complete\n") + printCheckSpecOutputHeader() + for _, o := range output { + printCheckSpecOutputBody(o) } - fmt.Println("Successfully validated check:spec") return nil } diff --git a/cmd/check_spec_utils.go b/cmd/check_spec_utils.go index c00cfc44..52d9e4d9 100644 --- a/cmd/check_spec_utils.go +++ b/cmd/check_spec_utils.go @@ -16,17 +16,25 @@ package cmd import ( "errors" + "fmt" + "github.com/coinbase/rosetta-sdk-go/fetcher" "github.com/coinbase/rosetta-sdk-go/types" + "github.com/fatih/color" ) var ( - errBlockIdentifier = errors.New("BlockIdentifier must have both index and hash") - errTimestamp = errors.New("Timestamp must be greater than or equal to 0") - errPeer = errors.New("Peer should be present and peer id should not be empty") - errVersion = errors.New("Version should not be empty") - errOperationStatus = errors.New("Operation status should not be empty") - errOperationTypeEmpty = errors.New("Operation type should not be empty") + errBlockIdentifierNullPointer = errors.New("Null pointer to BlockIdentifier object") + errBlockIdentifierEmptyHash = errors.New("BlockIdentifier can't have empty hash") + errBlockIdentifierNegativeIndex = errors.New("BlockIdentifier can't have negative index") + + errTimestampNegative = errors.New("Timestamp can't be negative") + errVersionEmpty = errors.New("Version can't be empty") + errVersionNullPointer = errors.New("Null pointer to Version object") + + errOperationStatusEmptyStatus = errors.New("OperationStatus can't have empty status value") + errOperationStatusNullPointer = errors.New("Null pointer to OperationStatus object") + errOperationTypeEmpty = errors.New("OperationType can't be empty") errErrorEmpty = errors.New("Error object can't be empty") errErrorEmptyMessage = errors.New("Error object can't have empty message") @@ -34,39 +42,90 @@ var ( errCallMethodEmptyName = errors.New("Call method name can't be empty") errBalanceExemptionEmpty = errors.New("Balance exemption can't be empty") + errAllowNullPointer = errors.New("Null pointer to Allow object") + + errBalanceEmptyValue = errors.New("Amount can't be empty") + errCurrencyEmptySymbol = errors.New("Currency can't have empty symbol") + errCurrencyNegativeDecimals = errors.New("Currency can't have negative decimals") + errAccountNullPointer = errors.New("Null pointer to Account object") + + errCoinIdentifierEmpty = errors.New("Coin identifier can't be empty") + errCoinIdentifierNullPointer = errors.New("Null pointer to coin identifier object") + errBlockNotIdempotent = errors.New("Multiple calls with the same hash don't return the same block") + errBlockTip = errors.New("Unspecified block_identifier doesn't give the tip block") + errRosettaConfigNoConstruction = errors.New("No construction element in Rosetta config") +) + +type checkSpecAPI string +type checkSpecRequirement string +type checkSpecStatus string - errAllowNullPointer = errors.New("Null pointer to Allow object") +const ( + networkList checkSpecAPI = "/network/list" + networkOptions checkSpecAPI = "/network/options" + networkStatus checkSpecAPI = "/network/status" + accountBalance checkSpecAPI = "/account/balance" + accountCoins checkSpecAPI = "/account/coins" + block checkSpecAPI = "/block" + errorObject checkSpecAPI = "error object" + modes checkSpecAPI = "modes" + + networkIDs checkSpecRequirement = "network_identifiers is required" + offlineMode checkSpecRequirement = "endpoint should work in offline mode" + staticNetworkID checkSpecRequirement = "network_identifier must be static" + version checkSpecRequirement = "field version is required" + allow checkSpecRequirement = "field allow is required" + + currentBlockID checkSpecRequirement = "current_block_identifier is required" + currentBlockTime checkSpecRequirement = "current_block_timestamp is required" + genesisBlockID checkSpecRequirement = "genesis_block_identifier is required" + + blockID checkSpecRequirement = "block_identifier is required" + balances checkSpecRequirement = "field balances is required" + coins checkSpecRequirement = "field coins is required" + idempotent checkSpecRequirement = "same hash should return the same block" + defaultTip checkSpecRequirement = "tip should be returned if block_identifier is not specified" + + errorCode checkSpecRequirement = "error code is required" + errorMessage checkSpecRequirement = "error message is required" + diffURLs checkSpecRequirement = "offline_url should be different from offline_url and not empty" + + checkSpecSuccess checkSpecStatus = "Success" + checkSpecFailure checkSpecStatus = "Failure" ) +type checkSpecOutput struct { + api checkSpecAPI + validation map[checkSpecRequirement]checkSpecStatus +} + func validateBlockIdentifier(blockID *types.BlockIdentifier) error { - if blockID != nil && blockID.Hash != "" && blockID.Index >= 0 { - return nil + if blockID == nil { + return errBlockIdentifierNullPointer } - return errBlockIdentifier -} + if isEmpty(blockID.Hash) { + return errBlockIdentifierEmptyHash + } -func validateTimestamp(time int64) error { - if time >= 0 { - return nil + if isNegative(blockID.Index) { + return errBlockIdentifierNegativeIndex } - return errTimestamp + return nil } -func validatePeers(peers []*types.Peer) error { - for _, p := range peers { - if p == nil || p.PeerID == "" { - return errPeer - } +func validateTimestamp(time int64) error { + if isNegative(time) { + return errTimestampNegative } return nil } func validateVersion(version string) error { - if version == "" { - return errVersion + if isEmpty(version) { + return errVersionEmpty } return nil @@ -74,8 +133,12 @@ func validateVersion(version string) error { func validateOperationStatuses(oss []*types.OperationStatus) error { for _, os := range oss { - if os == nil || os.Status == "" { - return errOperationStatus + if os == nil { + return errOperationStatusNullPointer + } + + if isEmpty(os.Status) { + return errOperationStatusEmptyStatus } } @@ -84,7 +147,7 @@ func validateOperationStatuses(oss []*types.OperationStatus) error { func validateOperationTypes(ots []string) error { for _, ot := range ots { - if ot == "" { + if isEmpty(ot) { return errOperationTypeEmpty } } @@ -98,11 +161,11 @@ func validateErrors(errors []*types.Error) error { return errErrorEmpty } - if err.Code < 0 { + if isNegative(int64(err.Code)) { return errErrorNegativeCode } - if err.Message == "" { + if isEmpty(err.Message) { return errErrorEmptyMessage } } @@ -112,7 +175,7 @@ func validateErrors(errors []*types.Error) error { func validateCallMethods(methods []string) error { for _, m := range methods { - if m == "" { + if isEmpty(m) { return errCallMethodEmptyName } } @@ -129,3 +192,157 @@ func validateBalanceExemptions(exs []*types.BalanceExemption) error { return nil } + +func validateBalances(balances []*types.Amount) error { + for _, b := range balances { + if err := validateBalance(b); err != nil { + return err + } + } + + return nil +} + +func validateBalance(amt *types.Amount) error { + if isEmpty(amt.Value) { + return errBalanceEmptyValue + } + + if err := validateCurrency(amt.Currency); err != nil { + return err + } + + return nil +} + +func validateCurrency(currency *types.Currency) error { + if isEmpty(currency.Symbol) { + return errCurrencyEmptySymbol + } + + if isNegative(int64(currency.Decimals)) { + return errCurrencyNegativeDecimals + } + + return nil +} + +func validateCoins(coins []*types.Coin) error { + for _, coin := range coins { + if err := validateCoinIdentifier(coin.CoinIdentifier); err != nil { + return err + } + if err := validateBalance(coin.Amount); err != nil { + return err + } + } + + return nil +} + +func validateCoinIdentifier(coinID *types.CoinIdentifier) error { + if coinID == nil { + return errCoinIdentifierNullPointer + } + + if isEmpty(coinID.Identifier) { + return errCoinIdentifierEmpty + } + + return nil +} + +func twoModes() checkSpecOutput { + output := checkSpecOutput{ + api: modes, + validation: map[checkSpecRequirement]checkSpecStatus{ + diffURLs: checkSpecSuccess, + }, + } + + if isEmpty(Config.OnlineURL) || + isEmpty(Config.Construction.OfflineURL) || + isEqual(Config.OnlineURL, Config.Construction.OfflineURL) { + setValidationStatus(output, diffURLs, checkSpecFailure) + } + + return output +} + +func markAllValidationsFailed(output checkSpecOutput) { + for k := range output.validation { + output.validation[k] = checkSpecFailure + } +} + +func setValidationStatus(output checkSpecOutput, req checkSpecRequirement, status checkSpecStatus) { + output.validation[req] = status +} + +func validateErrorObject(err *fetcher.Error, output checkSpecOutput) { + if err != nil { + if err.ClientErr != nil && isNegative(int64(err.ClientErr.Code)) { + printError("%v\n", errErrorNegativeCode) + setValidationStatus(output, errorCode, checkSpecFailure) + } + + if err.ClientErr != nil && isEmpty(err.ClientErr.Message) { + printError("%v\n", errErrorEmptyMessage) + setValidationStatus(output, errorMessage, checkSpecFailure) + } + } +} + +func printInfo(format string, a ...interface{}) { + fmt.Printf(format, a...) +} + +func printError(format string, a ...interface{}) { + fmt.Print(color.RedString(format, a...)) +} + +func printSuccess(format string, a ...interface{}) { + fmt.Print(color.GreenString(format, a...)) +} + +func printValidationResult(format string, status checkSpecStatus, a ...interface{}) { + if status == checkSpecFailure { + printError(format, a...) + } else { + printSuccess(format, a...) + } +} + +func printCheckSpecOutputHeader() { + printInfo("%v\n", "+--------------------------+-------------------------------------------------------------------+-----------+") + printInfo("%v\n", "| API | Requirement | Status |") + printInfo("%v\n", "+--------------------------+-------------------------------------------------------------------+-----------+") +} + +func printCheckSpecOutputBody(output checkSpecOutput) { + for k, v := range output.validation { + // print api + printInfo("%v", "| ") + printValidationResult("%v", v, output.api) + for j := 0; j < 24-len(output.api); j++ { + printInfo("%v", " ") + } + + // print requirement description + printInfo("%v", "| ") + printValidationResult("%v", v, k) + for j := 0; j < 65-len(k); j++ { + printInfo(" ") + } + + // print validation status + printInfo("%v", "| ") + printValidationResult("%v", v, v) + for j := 0; j < 9-len(v); j++ { + printInfo("%v", " ") + } + + printInfo("%v\n", "|") + printInfo("%v\n", "+--------------------------+-------------------------------------------------------------------+-----------+") + } +} diff --git a/cmd/utils_shared.go b/cmd/utils_shared.go new file mode 100644 index 00000000..97147088 --- /dev/null +++ b/cmd/utils_shared.go @@ -0,0 +1,31 @@ +// Copyright 2022 Coinbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package cmd + +func isEmpty(s string) bool { + return s == "" +} + +func isNegative(n int64) bool { + return n < 0 +} + +func isEqual(s1 string, s2 string) bool { + return s1 == s2 +} + +func isUTXO() bool { + return Config.CoinSupported +} From 7df8438916e581dd10da4f9529ae6650e09c605d Mon Sep 17 00:00:00 2001 From: shiatcb Date: Mon, 27 Jun 2022 12:56:06 -0400 Subject: [PATCH 327/376] ROSE-420: clean up check:spec logic to avoid duplicate validations --- cmd/check_spec.go | 176 +++------------------------------- cmd/check_spec_utils.go | 203 ++-------------------------------------- 2 files changed, 22 insertions(+), 357 deletions(-) diff --git a/cmd/check_spec.go b/cmd/check_spec.go index fbd04983..fdcda28b 100644 --- a/cmd/check_spec.go +++ b/cmd/check_spec.go @@ -104,100 +104,15 @@ func (cs *checkSpec) networkOptions(ctx context.Context) checkSpecOutput { } defer printInfo("/network/options validated\n") - res, err := cs.offlineFetcher.NetworkOptionsRetry(ctx, Config.Network, nil) + // NetworkOptionsRetry handles validation of /network/options response + // This is an endpoint for offline mode + _, err := cs.offlineFetcher.NetworkOptionsRetry(ctx, Config.Network, nil) if err != nil { printError("%v: unable to fetch network options\n", err.Err) markAllValidationsFailed(output) return output } - // version is required - if res.Version == nil { - setValidationStatus(output, version, checkSpecFailure) - printError("%v: unable to find version in /network/options response\n", errVersionNullPointer) - } - - if err := validateVersion(res.Version.RosettaVersion); err != nil { - setValidationStatus(output, version, checkSpecFailure) - printError("%v\n", err) - } - - if err := validateVersion(res.Version.NodeVersion); err != nil { - setValidationStatus(output, version, checkSpecFailure) - printError("%v\n", err) - } - - // allow is required - if res.Allow == nil { - setValidationStatus(output, allow, checkSpecFailure) - printError("%v: unable to find allow in /network/options response\n", errAllowNullPointer) - } - - if err := validateOperationStatuses(res.Allow.OperationStatuses); err != nil { - setValidationStatus(output, allow, checkSpecFailure) - printError("%v\n", err) - } - - if err := validateOperationTypes(res.Allow.OperationTypes); err != nil { - setValidationStatus(output, allow, checkSpecFailure) - printError("%v\n", err) - } - - if err := validateErrors(res.Allow.Errors); err != nil { - setValidationStatus(output, allow, checkSpecFailure) - printError("%v\n", err) - } - - if err := validateCallMethods(res.Allow.CallMethods); err != nil { - setValidationStatus(output, allow, checkSpecFailure) - printError("%v\n", err) - } - - if err := validateBalanceExemptions(res.Allow.BalanceExemptions); err != nil { - setValidationStatus(output, allow, checkSpecFailure) - printError("%v\n", err) - } - - return output -} - -func (cs *checkSpec) networkStatus(ctx context.Context) checkSpecOutput { - printInfo("validating /network/status ...\n") - output := checkSpecOutput{ - api: networkStatus, - validation: map[checkSpecRequirement]checkSpecStatus{ - currentBlockID: checkSpecSuccess, - currentBlockTime: checkSpecSuccess, - genesisBlockID: checkSpecSuccess, - }, - } - defer printInfo("/network/status validated\n") - - res, err := cs.onlineFetcher.NetworkStatusRetry(ctx, Config.Network, nil) - if err != nil { - printError("%v: unable to fetch network status\n", err.Err) - markAllValidationsFailed(output) - return output - } - - // current_block_identifier is required - if err := validateBlockIdentifier(res.CurrentBlockIdentifier); err != nil { - printError("%v\n", err) - setValidationStatus(output, currentBlockID, checkSpecFailure) - } - - // current_block_timestamp is required - if err := validateTimestamp(res.CurrentBlockTimestamp); err != nil { - printError("%v\n", err) - setValidationStatus(output, currentBlockTime, checkSpecFailure) - } - - // genesis_block_identifier is required - if err := validateBlockIdentifier(res.GenesisBlockIdentifier); err != nil { - printError("%v\n", err) - setValidationStatus(output, genesisBlockID, checkSpecFailure) - } - return output } @@ -213,7 +128,8 @@ func (cs *checkSpec) networkList(ctx context.Context) checkSpecOutput { } defer printInfo("/network/list validated\n") - networks, err := cs.offlineFetcher.NetworkList(ctx, nil) + // endpoint for offline mode + networks, err := cs.offlineFetcher.NetworkListRetry(ctx, nil) if err != nil { printError("%v: unable to fetch network list", err.Err) markAllValidationsFailed(output) @@ -222,7 +138,7 @@ func (cs *checkSpec) networkList(ctx context.Context) checkSpecOutput { if len(networks.NetworkIdentifiers) == 0 { printError("network_identifiers is required") - setValidationStatus(output, networkIDs, checkSpecFailure) + setValidationStatusFailed(output, networkIDs) } for _, network := range networks.NetworkIdentifiers { @@ -232,59 +148,9 @@ func (cs *checkSpec) networkList(ctx context.Context) checkSpecOutput { } } + // static network ID printError("network_identifier in configuration file is not returned by /network/list") - setValidationStatus(output, staticNetworkID, checkSpecFailure) - return output -} - -func (cs *checkSpec) accountBalance(ctx context.Context) checkSpecOutput { - printInfo("validating /account/balance ...\n") - output := checkSpecOutput{ - api: accountBalance, - validation: map[checkSpecRequirement]checkSpecStatus{ - blockID: checkSpecSuccess, - balances: checkSpecSuccess, - }, - } - defer printInfo("/account/balance validated\n") - - acct, partBlockID, currencies, err := cs.getAccount(ctx) - if err != nil { - markAllValidationsFailed(output) - printError("%v: unable to get an account\n", err) - return output - } - if acct == nil { - markAllValidationsFailed(output) - printError("%v\n", errAccountNullPointer) - return output - } - - // fetch account balance - block, amt, _, fetchErr := cs.onlineFetcher.AccountBalanceRetry( - ctx, - Config.Network, - acct, - partBlockID, - currencies) - if err != nil { - markAllValidationsFailed(output) - printError("%v: unable to fetch balance for account: %v\n", fetchErr.Err, *acct) - return output - } - - // block_identifier is required - if err := validateBlockIdentifier(block); err != nil { - printError("%v\n", err) - setValidationStatus(output, blockID, checkSpecFailure) - } - - // balances is required - if err := validateBalances(amt); err != nil { - printError("%v\n", err) - setValidationStatus(output, balances, checkSpecFailure) - } - + setValidationStatusFailed(output, staticNetworkID) return output } @@ -312,7 +178,7 @@ func (cs *checkSpec) accountCoins(ctx context.Context) checkSpecOutput { return output } - block, cs, _, fetchErr := cs.onlineFetcher.AccountCoinsRetry( + _, _, _, fetchErr := cs.onlineFetcher.AccountCoinsRetry( ctx, Config.Network, acct, @@ -323,20 +189,6 @@ func (cs *checkSpec) accountCoins(ctx context.Context) checkSpecOutput { markAllValidationsFailed(output) return output } - - // block_identifier is required - err = validateBlockIdentifier(block) - if err != nil { - printError("%v\n", err) - setValidationStatus(output, blockID, checkSpecFailure) - } - - // coins is required - err = validateCoins(cs) - if err != nil { - printError("%v\n", err) - setValidationStatus(output, coins, checkSpecFailure) - } } return output @@ -380,7 +232,7 @@ func (cs *checkSpec) block(ctx context.Context) checkSpecOutput { block = b } else if !isEqual(types.Hash(*block), types.Hash(*b)) { printError("%v\n", errBlockNotIdempotent) - setValidationStatus(output, idempotent, checkSpecFailure) + setValidationStatusFailed(output, idempotent) } } @@ -388,7 +240,7 @@ func (cs *checkSpec) block(ctx context.Context) checkSpecOutput { res, fetchErr = cs.onlineFetcher.NetworkStatusRetry(ctx, Config.Network, nil) if fetchErr != nil { printError("%v: unable to get network status\n", fetchErr.Err) - setValidationStatus(output, defaultTip, checkSpecFailure) + setValidationStatusFailed(output, defaultTip) return output } tip = res.CurrentBlockIdentifier @@ -398,14 +250,14 @@ func (cs *checkSpec) block(ctx context.Context) checkSpecOutput { block, fetchErr = cs.onlineFetcher.BlockRetry(ctx, Config.Network, emptyBlockID) if fetchErr != nil { printError("%v: unable to fetch tip block\n", fetchErr.Err) - setValidationStatus(output, defaultTip, checkSpecFailure) + setValidationStatusFailed(output, defaultTip) return output } // block index returned from /block should be >= the index returned by /network/status if isNegative(block.BlockIdentifier.Index - tip.Index) { printError("%v\n", errBlockTip) - setValidationStatus(output, defaultTip, checkSpecFailure) + setValidationStatusFailed(output, defaultTip) } return output @@ -514,10 +366,8 @@ func runCheckSpecCmd(_ *cobra.Command, _ []string) error { output := []checkSpecOutput{} // validate api endpoints - output = append(output, cs.networkStatus(ctx)) output = append(output, cs.networkList(ctx)) output = append(output, cs.networkOptions(ctx)) - output = append(output, cs.accountBalance(ctx)) output = append(output, cs.accountCoins(ctx)) output = append(output, cs.block(ctx)) output = append(output, cs.errorObject(ctx)) diff --git a/cmd/check_spec_utils.go b/cmd/check_spec_utils.go index 52d9e4d9..ebbb26cc 100644 --- a/cmd/check_spec_utils.go +++ b/cmd/check_spec_utils.go @@ -19,38 +19,13 @@ import ( "fmt" "github.com/coinbase/rosetta-sdk-go/fetcher" - "github.com/coinbase/rosetta-sdk-go/types" "github.com/fatih/color" ) var ( - errBlockIdentifierNullPointer = errors.New("Null pointer to BlockIdentifier object") - errBlockIdentifierEmptyHash = errors.New("BlockIdentifier can't have empty hash") - errBlockIdentifierNegativeIndex = errors.New("BlockIdentifier can't have negative index") - - errTimestampNegative = errors.New("Timestamp can't be negative") - errVersionEmpty = errors.New("Version can't be empty") - errVersionNullPointer = errors.New("Null pointer to Version object") - - errOperationStatusEmptyStatus = errors.New("OperationStatus can't have empty status value") - errOperationStatusNullPointer = errors.New("Null pointer to OperationStatus object") - errOperationTypeEmpty = errors.New("OperationType can't be empty") - - errErrorEmpty = errors.New("Error object can't be empty") - errErrorEmptyMessage = errors.New("Error object can't have empty message") - errErrorNegativeCode = errors.New("Error object can't have negative code") - - errCallMethodEmptyName = errors.New("Call method name can't be empty") - errBalanceExemptionEmpty = errors.New("Balance exemption can't be empty") - errAllowNullPointer = errors.New("Null pointer to Allow object") - - errBalanceEmptyValue = errors.New("Amount can't be empty") - errCurrencyEmptySymbol = errors.New("Currency can't have empty symbol") - errCurrencyNegativeDecimals = errors.New("Currency can't have negative decimals") - errAccountNullPointer = errors.New("Null pointer to Account object") - - errCoinIdentifierEmpty = errors.New("Coin identifier can't be empty") - errCoinIdentifierNullPointer = errors.New("Null pointer to coin identifier object") + errErrorEmptyMessage = errors.New("Error object can't have empty message") + errErrorNegativeCode = errors.New("Error object can't have negative code") + errAccountNullPointer = errors.New("Null pointer to Account object") errBlockNotIdempotent = errors.New("Multiple calls with the same hash don't return the same block") errBlockTip = errors.New("Unspecified block_identifier doesn't give the tip block") errRosettaConfigNoConstruction = errors.New("No construction element in Rosetta config") @@ -63,8 +38,6 @@ type checkSpecStatus string const ( networkList checkSpecAPI = "/network/list" networkOptions checkSpecAPI = "/network/options" - networkStatus checkSpecAPI = "/network/status" - accountBalance checkSpecAPI = "/account/balance" accountCoins checkSpecAPI = "/account/coins" block checkSpecAPI = "/block" errorObject checkSpecAPI = "error object" @@ -76,12 +49,7 @@ const ( version checkSpecRequirement = "field version is required" allow checkSpecRequirement = "field allow is required" - currentBlockID checkSpecRequirement = "current_block_identifier is required" - currentBlockTime checkSpecRequirement = "current_block_timestamp is required" - genesisBlockID checkSpecRequirement = "genesis_block_identifier is required" - blockID checkSpecRequirement = "block_identifier is required" - balances checkSpecRequirement = "field balances is required" coins checkSpecRequirement = "field coins is required" idempotent checkSpecRequirement = "same hash should return the same block" defaultTip checkSpecRequirement = "tip should be returned if block_identifier is not specified" @@ -99,159 +67,6 @@ type checkSpecOutput struct { validation map[checkSpecRequirement]checkSpecStatus } -func validateBlockIdentifier(blockID *types.BlockIdentifier) error { - if blockID == nil { - return errBlockIdentifierNullPointer - } - - if isEmpty(blockID.Hash) { - return errBlockIdentifierEmptyHash - } - - if isNegative(blockID.Index) { - return errBlockIdentifierNegativeIndex - } - - return nil -} - -func validateTimestamp(time int64) error { - if isNegative(time) { - return errTimestampNegative - } - - return nil -} - -func validateVersion(version string) error { - if isEmpty(version) { - return errVersionEmpty - } - - return nil -} - -func validateOperationStatuses(oss []*types.OperationStatus) error { - for _, os := range oss { - if os == nil { - return errOperationStatusNullPointer - } - - if isEmpty(os.Status) { - return errOperationStatusEmptyStatus - } - } - - return nil -} - -func validateOperationTypes(ots []string) error { - for _, ot := range ots { - if isEmpty(ot) { - return errOperationTypeEmpty - } - } - - return nil -} - -func validateErrors(errors []*types.Error) error { - for _, err := range errors { - if err == nil { - return errErrorEmpty - } - - if isNegative(int64(err.Code)) { - return errErrorNegativeCode - } - - if isEmpty(err.Message) { - return errErrorEmptyMessage - } - } - - return nil -} - -func validateCallMethods(methods []string) error { - for _, m := range methods { - if isEmpty(m) { - return errCallMethodEmptyName - } - } - - return nil -} - -func validateBalanceExemptions(exs []*types.BalanceExemption) error { - for _, e := range exs { - if e == nil { - return errBalanceExemptionEmpty - } - } - - return nil -} - -func validateBalances(balances []*types.Amount) error { - for _, b := range balances { - if err := validateBalance(b); err != nil { - return err - } - } - - return nil -} - -func validateBalance(amt *types.Amount) error { - if isEmpty(amt.Value) { - return errBalanceEmptyValue - } - - if err := validateCurrency(amt.Currency); err != nil { - return err - } - - return nil -} - -func validateCurrency(currency *types.Currency) error { - if isEmpty(currency.Symbol) { - return errCurrencyEmptySymbol - } - - if isNegative(int64(currency.Decimals)) { - return errCurrencyNegativeDecimals - } - - return nil -} - -func validateCoins(coins []*types.Coin) error { - for _, coin := range coins { - if err := validateCoinIdentifier(coin.CoinIdentifier); err != nil { - return err - } - if err := validateBalance(coin.Amount); err != nil { - return err - } - } - - return nil -} - -func validateCoinIdentifier(coinID *types.CoinIdentifier) error { - if coinID == nil { - return errCoinIdentifierNullPointer - } - - if isEmpty(coinID.Identifier) { - return errCoinIdentifierEmpty - } - - return nil -} - func twoModes() checkSpecOutput { output := checkSpecOutput{ api: modes, @@ -263,7 +78,7 @@ func twoModes() checkSpecOutput { if isEmpty(Config.OnlineURL) || isEmpty(Config.Construction.OfflineURL) || isEqual(Config.OnlineURL, Config.Construction.OfflineURL) { - setValidationStatus(output, diffURLs, checkSpecFailure) + setValidationStatusFailed(output, diffURLs) } return output @@ -275,20 +90,20 @@ func markAllValidationsFailed(output checkSpecOutput) { } } -func setValidationStatus(output checkSpecOutput, req checkSpecRequirement, status checkSpecStatus) { - output.validation[req] = status +func setValidationStatusFailed(output checkSpecOutput, req checkSpecRequirement) { + output.validation[req] = checkSpecFailure } func validateErrorObject(err *fetcher.Error, output checkSpecOutput) { if err != nil { if err.ClientErr != nil && isNegative(int64(err.ClientErr.Code)) { printError("%v\n", errErrorNegativeCode) - setValidationStatus(output, errorCode, checkSpecFailure) + setValidationStatusFailed(output, errorCode) } if err.ClientErr != nil && isEmpty(err.ClientErr.Message) { printError("%v\n", errErrorEmptyMessage) - setValidationStatus(output, errorMessage, checkSpecFailure) + setValidationStatusFailed(output, errorMessage) } } } @@ -315,7 +130,7 @@ func printValidationResult(format string, status checkSpecStatus, a ...interface func printCheckSpecOutputHeader() { printInfo("%v\n", "+--------------------------+-------------------------------------------------------------------+-----------+") - printInfo("%v\n", "| API | Requirement | Status |") + printInfo("%v\n", "| API | Requirement | Status |") printInfo("%v\n", "+--------------------------+-------------------------------------------------------------------+-----------+") } From 4cebc382b29a5bbdb8e532fd24e3e1e4b741c57c Mon Sep 17 00:00:00 2001 From: Jingfu Wang Date: Mon, 27 Jun 2022 16:07:57 -0400 Subject: [PATCH 328/376] feat: destination tag support Signed-off-by: Jingfu Wang --- .gitignore | 3 +++ configuration/configuration.go | 10 ++++++---- go.mod | 3 +-- go.sum | 10 +++++----- pkg/processor/broadcast_storage_handler.go | 10 ++++++++++ pkg/processor/coordinator_helper.go | 3 +++ 6 files changed, 28 insertions(+), 11 deletions(-) diff --git a/.gitignore b/.gitignore index 40fb1133..c8acf91f 100644 --- a/.gitignore +++ b/.gitignore @@ -6,3 +6,6 @@ bin/ # Mac files .DS_Store + +# VS Code setting +.vscode \ No newline at end of file diff --git a/configuration/configuration.go b/configuration/configuration.go index 3b685a4e..d1a392c7 100644 --- a/configuration/configuration.go +++ b/configuration/configuration.go @@ -266,10 +266,12 @@ func assertConstructionConfiguration(ctx context.Context, config *ConstructionCo return fmt.Errorf("Account.Address is missing for prefunded account") } - // Check if valid Currency - err = asserter.Currency(account.Currency) - if err != nil { - return fmt.Errorf("%w: invalid currency for prefunded account", err) + // Check if valid Currency when Currency is specified + if account.Currency != nil { + err = asserter.Currency(account.Currency) + if err != nil { + return fmt.Errorf("%w: invalid currency for prefunded account", err) + } } } diff --git a/go.mod b/go.mod index e9271db5..4630a248 100644 --- a/go.mod +++ b/go.mod @@ -11,9 +11,8 @@ require ( github.com/olekukonko/tablewriter v0.0.5 github.com/pkg/errors v0.9.1 github.com/spf13/cobra v1.4.0 - github.com/stretchr/testify v1.7.1 + github.com/stretchr/testify v1.7.2 go.uber.org/zap v1.21.0 - golang.org/x/crypto v0.0.0-20210817164053-32db794688a5 // indirect golang.org/x/sync v0.0.0-20210220032951-036812b2e83c golang.org/x/sys v0.0.0-20211205182925-97ca703d548d // indirect google.golang.org/protobuf v1.27.1 // indirect diff --git a/go.sum b/go.sum index 314f143a..6d364c01 100644 --- a/go.sum +++ b/go.sum @@ -420,8 +420,8 @@ github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81P github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.7.1 h1:5TQK59W5E3v0r2duFAb7P95B6hEeOyEnHRa8MjYSMTY= -github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.2 h1:4jaiDzPyXQvSd7D0EjG45355tLlV3VOECpq10pLC+8s= +github.com/stretchr/testify v1.7.2/go.mod h1:R6va5+xMeoiuVRoj+gSkQ7d3FALtqAAGI1FQKckRals= github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7/go.mod h1:q4W45IWZaF22tdD+VEXcAWRA037jwmWEB5VWYORlTpc= github.com/tidwall/gjson v1.12.1/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= github.com/tidwall/gjson v1.14.0 h1:6aeJ0bzojgWLa82gDQHcx3S0Lr/O51I9bJ5nv6JFx5w= @@ -481,9 +481,8 @@ golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPh golang.org/x/crypto v0.0.0-20200820211705-5c72a883971a/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20201221181555-eec23a3978ad/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= +golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97 h1:/UOmuWzQfxxo9UtlXMwuQU8CMgg1eZXqTRwkSQJWKOI= golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.0.0-20210817164053-32db794688a5 h1:HWj/xjIHfjYU5nVXpTM0s39J9CbLn7Cc5a7IC5rwsMQ= -golang.org/x/crypto v0.0.0-20210817164053-32db794688a5/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20180807140117-3d87b88a115f/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= @@ -723,8 +722,9 @@ gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo= gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= diff --git a/pkg/processor/broadcast_storage_handler.go b/pkg/processor/broadcast_storage_handler.go index 32e617f9..25ee8ee8 100644 --- a/pkg/processor/broadcast_storage_handler.go +++ b/pkg/processor/broadcast_storage_handler.go @@ -18,6 +18,7 @@ import ( "context" "fmt" "math/big" + "reflect" "github.com/coinbase/rosetta-cli/configuration" @@ -67,6 +68,7 @@ func (h *BroadcastStorageHandler) TransactionConfirmed( blockIdentifier *types.BlockIdentifier, transaction *types.Transaction, intent []*types.Operation, + intentMetadata map[string]interface{}, ) error { _, _, relatedTransactions, err := h.blockStorage.FindRelatedTransactions(ctx, transaction.TransactionIdentifier, dbTx) if err != nil { @@ -82,6 +84,14 @@ func (h *BroadcastStorageHandler) TransactionConfirmed( return fmt.Errorf("%w: confirmed transaction did not match intent", err) } + // Validate destination memo if it's needed + if intentMemo, found := intentMetadata["memo"]; found { + observedMemo := transaction.Metadata["memo"] + if !reflect.DeepEqual(intentMemo, observedMemo) { + return fmt.Errorf("observed destination memo did not match intent destination memo, observed destination memo: %v, intent destination memo: %v", observedMemo, intentMemo) + } + } + _, _ = h.counterStorage.UpdateTransactional( ctx, dbTx, diff --git a/pkg/processor/coordinator_helper.go b/pkg/processor/coordinator_helper.go index 6d204fc3..496611bd 100644 --- a/pkg/processor/coordinator_helper.go +++ b/pkg/processor/coordinator_helper.go @@ -473,12 +473,14 @@ func (c *CoordinatorHelper) Broadcast( transactionIdentifier *types.TransactionIdentifier, payload string, confirmationDepth int64, + transactionMetadata map[string]interface{}, ) error { c.verboseLog(queue, constructionSubmit, arg{argNetwork, network}, arg{argIntent, intent}, arg{argTransactionIdentifier, transactionIdentifier}, arg{argNetworkTransaction, payload}, + arg{argMetadata, transactionMetadata}, ) return c.broadcastStorage.Broadcast( ctx, @@ -489,6 +491,7 @@ func (c *CoordinatorHelper) Broadcast( transactionIdentifier, payload, confirmationDepth, + transactionMetadata, ) } From 2e74351c7e7ee7ca103aa05a9dcf0e2ec024f0a6 Mon Sep 17 00:00:00 2001 From: racbc Date: Wed, 29 Jun 2022 13:19:34 -0400 Subject: [PATCH 329/376] (nit) Update check_data long description to match docs (#330) * Update check_data long description to match docs * Fix copyright year in cmd/check_data.go --- cmd/check_data.go | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/cmd/check_data.go b/cmd/check_data.go index eb79d77c..cc54433b 100644 --- a/cmd/check_data.go +++ b/cmd/check_data.go @@ -1,4 +1,4 @@ -// Copyright 2020 Coinbase, Inc. +// Copyright 2022 Coinbase, Inc. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -32,35 +32,35 @@ var ( checkDataCmd = &cobra.Command{ Use: "check:data", Short: "Check the correctness of a Rosetta Data API Implementation", - Long: `Check all server responses are properly constructed, that -there are no duplicate blocks and transactions, that blocks can be processed + Long: `Check all server responses are +properly constructed, that there are no duplicate blocks and transactions, that blocks can be processed from genesis to the current block (re-orgs handled automatically), and that computed balance changes are equal to balance changes reported by the node. - + When re-running this command, it will start where it left off if you specify some data directory. Otherwise, it will create a new temporary directory and start again from the genesis block. If you want to discard some number of blocks -populate the start_index filed in the configuration file with some block index. -Starting from a given index can be useful to debug a small range of blocks for -issues but it is highly recommended you sync from start to finish to ensure -all correctness checks are performed. - +populate the --start flag with some block index. Starting from a given index +can be useful to debug a small range of blocks for issues but it is highly +recommended you sync from start to finish to ensure all correctness checks +are performed. + By default, account balances are looked up at specific heights (instead of -only at the current block). If your node does not support this functionality, -you can disable historical balance lookups in your configuration file. This will -make reconciliation much less efficient but it will still work. - +only at the current block). If your node does not support this functionality +set historical balance disabled to true. This will make reconciliation much +less efficient but it will still work. + If check fails due to an INACTIVE reconciliation error (balance changed without any corresponding operation), the cli will automatically try to find the block missing an operation. If historical balance disabled is true, this automatic debugging tool does not work. - + To debug an INACTIVE account reconciliation error without historical balance lookup, set the interesting accounts to the path of a JSON file containing accounts that will be actively checked for balance changes at each block. This will return an error at the block where a balance change occurred with no corresponding operations. - + If your blockchain has a genesis allocation of funds and you set historical balance disabled to true, you must provide an absolute path to a JSON file containing initial balances with the From a520eec154f07d7a70f2a0cc5d1b693c1a8a1321 Mon Sep 17 00:00:00 2001 From: Jingfu Wang Date: Wed, 29 Jun 2022 16:20:19 -0400 Subject: [PATCH 330/376] chore: address comments Signed-off-by: Jingfu Wang --- .gitignore | 9 --------- configuration/configuration.go | 2 ++ pkg/processor/broadcast_storage_handler.go | 9 ++++++--- 3 files changed, 8 insertions(+), 12 deletions(-) diff --git a/.gitignore b/.gitignore index c8acf91f..218fd2fa 100644 --- a/.gitignore +++ b/.gitignore @@ -1,11 +1,2 @@ rosetta-cli bin/ - -# Intellij setting -.idea/ - -# Mac files -.DS_Store - -# VS Code setting -.vscode \ No newline at end of file diff --git a/configuration/configuration.go b/configuration/configuration.go index d1a392c7..70fb9c9d 100644 --- a/configuration/configuration.go +++ b/configuration/configuration.go @@ -267,6 +267,8 @@ func assertConstructionConfiguration(ctx context.Context, config *ConstructionCo } // Check if valid Currency when Currency is specified + // If Currency is not specified, the balances of all available currencies + // for the specific pre-funded account will be stored in the balance storage if account.Currency != nil { err = asserter.Currency(account.Currency) if err != nil { diff --git a/pkg/processor/broadcast_storage_handler.go b/pkg/processor/broadcast_storage_handler.go index 25ee8ee8..edb57c1e 100644 --- a/pkg/processor/broadcast_storage_handler.go +++ b/pkg/processor/broadcast_storage_handler.go @@ -86,9 +86,12 @@ func (h *BroadcastStorageHandler) TransactionConfirmed( // Validate destination memo if it's needed if intentMemo, found := intentMetadata["memo"]; found { - observedMemo := transaction.Metadata["memo"] - if !reflect.DeepEqual(intentMemo, observedMemo) { - return fmt.Errorf("observed destination memo did not match intent destination memo, observed destination memo: %v, intent destination memo: %v", observedMemo, intentMemo) + if observedMemo, found := transaction.Metadata["memo"]; found { + if !reflect.DeepEqual(intentMemo, observedMemo) { + return fmt.Errorf("observed destination memo did not match intent destination memo, observed destination memo: %v, intent destination memo: %v", observedMemo, intentMemo) + } + } else { + return fmt.Errorf("observed destination memo did not found, observed destination memo: %v, intent destination memo: %v", observedMemo, intentMemo) } } From b0e1fd4b9d1d274815959b8578df6c9036770b82 Mon Sep 17 00:00:00 2001 From: Madhur Shrimal Date: Wed, 29 Jun 2022 13:36:08 -0700 Subject: [PATCH 331/376] update version (#334) --- Dockerfile | 2 +- cmd/root.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index edd3b8cc..4840968b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -31,7 +31,7 @@ RUN mkdir -p "$GOPATH/src" "$GOPATH/bin" && chmod -R 777 "$GOPATH" WORKDIR /go/src -ARG VERSION=v0.7.8 +ARG VERSION=v0.7.9 RUN git clone https://github.com/coinbase/rosetta-cli.git && \ cd rosetta-cli && \ git fetch --all --tags && \ diff --git a/cmd/root.go b/cmd/root.go index caf10515..c7d05643 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -397,6 +397,6 @@ var versionCmd = &cobra.Command{ Use: "version", Short: "Print rosetta-cli version", Run: func(cmd *cobra.Command, args []string) { - fmt.Println("v0.7.8") + fmt.Println("v0.7.9") }, } From 208c2342f6c6a8a95c7d95e76ea22ba3ba39ef3e Mon Sep 17 00:00:00 2001 From: Jingfu Wang Date: Wed, 29 Jun 2022 19:04:08 -0400 Subject: [PATCH 332/376] fix: CI tests Signed-off-by: Jingfu Wang --- go.mod | 2 +- go.sum | 42 +++++++++++++++++++++++++----------------- 2 files changed, 26 insertions(+), 18 deletions(-) diff --git a/go.mod b/go.mod index 4630a248..4db9cbf4 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/coinbase/rosetta-cli go 1.16 require ( - github.com/coinbase/rosetta-sdk-go v0.7.9 + github.com/coinbase/rosetta-sdk-go v0.7.11-0.20220629212620-136b591fb3f4 github.com/fatih/color v1.13.0 github.com/golang/protobuf v1.5.2 // indirect github.com/google/go-cmp v0.5.6 // indirect diff --git a/go.sum b/go.sum index 6d364c01..8797de37 100644 --- a/go.sum +++ b/go.sum @@ -28,8 +28,8 @@ github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v0.3.0/go.mod h1:tPaiy8S5bQ github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/DATA-DOG/go-sqlmock v1.3.3/go.mod h1:f/Ixk793poVmq4qj/V1dPUg2JEAKC73Q5eFN3EC/SaM= -github.com/DataDog/zstd v1.5.0 h1:+K/VEwIAaPcHiMtQvpLD4lqW7f0Gk3xdYZmI1hD+CXo= -github.com/DataDog/zstd v1.5.0/go.mod h1:g4AWEaM3yOg3HYfnJ3YIawPnVdXJh9QME85blwSAmyw= +github.com/DataDog/zstd v1.5.2 h1:vUG4lAyuPCXO0TLbXvPv7EB7cNK1QV/luu55UHLrrn8= +github.com/DataDog/zstd v1.5.2/go.mod h1:g4AWEaM3yOg3HYfnJ3YIawPnVdXJh9QME85blwSAmyw= github.com/OneOfOne/xxhash v1.2.2 h1:KMrpdQIwFcEqXDklaen+P1axHaj9BSKzvpUUfnHldSE= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= github.com/StackExchange/wmi v0.0.0-20180116203802-5d049714c4a6/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg= @@ -64,8 +64,7 @@ github.com/btcsuite/btcd v0.20.1-beta/go.mod h1:wVuoA8VJLEcwgqHBwHmzLRazpKxTv13P github.com/btcsuite/btcd v0.21.0-beta.0.20201114000516-e9c7a5ac6401/go.mod h1:Sv4JPQ3/M+teHz9Bo5jBpkNcP0x6r7rdihlNL/7tTAs= github.com/btcsuite/btcd v0.22.1 h1:CnwP9LM/M9xuRrGSCGeMVs9iv09uMqwsVX7EeIpgV2c= github.com/btcsuite/btcd v0.22.1/go.mod h1:wqgTSL29+50LRkmOVknEdmt8ZojIzhuWvgu/iptuN7Y= -github.com/btcsuite/btcd/btcec/v2 v2.1.2/go.mod h1:ctjw4H1kknNJmRN4iP1R7bTQ+v3GJkZBd6mui8ZsAZE= -github.com/btcsuite/btcd/chaincfg/chainhash v1.0.0/go.mod h1:7SFka0XMvUgj3hfZtydOrQY2mwhPclbT2snogU7SQQc= +github.com/btcsuite/btcd/btcec/v2 v2.2.0/go.mod h1:U7MHm051Al6XmscBQ0BoNydpOTsFAn707034b5nY8zU= github.com/btcsuite/btcd/chaincfg/chainhash v1.0.1 h1:q0rUy8C/TYNBQS1+CGKw68tLOFYSNEs0TFnxxnS9+4U= github.com/btcsuite/btcd/chaincfg/chainhash v1.0.1/go.mod h1:7SFka0XMvUgj3hfZtydOrQY2mwhPclbT2snogU7SQQc= github.com/btcsuite/btclog v0.0.0-20170628155309-84c8d2346e9f/go.mod h1:TdznJufoqS23FtqVCzL0ZqgP5MqXbb4fg/WgDys70nA= @@ -98,8 +97,8 @@ github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDk github.com/cloudflare/cloudflare-go v0.14.0/go.mod h1:EnwdgGMaFOruiPZRFSgn+TsQ3hQ7C/YWzIGLeu5c304= github.com/coinbase/kryptology v1.8.0 h1:Aoq4gdTsJhSU3lNWsD5BWmFSz2pE0GlmrljaOxepdYY= github.com/coinbase/kryptology v1.8.0/go.mod h1:RYXOAPdzOGUe3qlSFkMGn58i3xUA8hmxYHksuq+8ciI= -github.com/coinbase/rosetta-sdk-go v0.7.9 h1:lqllBjMnazTjIqYrOGv8h8jxjg9+hJazIGZr9ZvoCcA= -github.com/coinbase/rosetta-sdk-go v0.7.9/go.mod h1:0/knutI7XGVqXmmH4OQD8OckFrbQ8yMsUZTG7FXCR2M= +github.com/coinbase/rosetta-sdk-go v0.7.11-0.20220629212620-136b591fb3f4 h1:cHRCEzEk+4xerPtuJgO5a71qp7jIwGDKCKq+pte2VHA= +github.com/coinbase/rosetta-sdk-go v0.7.11-0.20220629212620-136b591fb3f4/go.mod h1:HLTqSTSnOGLWHGTxoUJQO2TLuKkas1B9i/7ByerK6lM= github.com/consensys/bavard v0.1.8-0.20210406032232-f3452dc9b572/go.mod h1:Bpd0/3mZuaj6Sj+PqrmIquiOKy397AKGThQPaGzNXAQ= github.com/consensys/bavard v0.1.8-0.20210915155054-088da2f7f54a/go.mod h1:9ItSMtA/dXMAiL7BG6bqW2m3NdSEObYWoH223nGHukI= github.com/consensys/gnark-crypto v0.4.1-0.20210426202927-39ac3d4b3f1f/go.mod h1:815PAHg3wvysy0SyIqanF8gZ0Y1wjk/hrDHD/iT88+Q= @@ -138,8 +137,8 @@ github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8 github.com/dlclark/regexp2 v1.4.1-0.20201116162257-a2a8dda75c91/go.mod h1:2pZnwuY/m+8K6iRw6wQdMtk+rH5tNGR1i55kozfMjCc= github.com/dnaeon/go-vcr v1.1.0/go.mod h1:M7tiix8f0r6mKKJ3Yq/kqU1OYf3MnfmBWVbPx/yU9ko= github.com/dnaeon/go-vcr v1.2.0/go.mod h1:R4UdLID7HZT3taECzJs4YgbbH6PIGXB6W/sc5OLb6RQ= -github.com/docker/docker v1.4.2-0.20180625184442-8e610b2b55bf/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= -github.com/dop251/goja v0.0.0-20211011172007-d99e4b8cbf48/go.mod h1:R9ET47fwRVRPZnOGvHxxhuZcbrMCuiqOz3Rlrh4KSnk= +github.com/docker/docker v1.6.2/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= +github.com/dop251/goja v0.0.0-20220405120441-9037c2b61cbf/go.mod h1:R9ET47fwRVRPZnOGvHxxhuZcbrMCuiqOz3Rlrh4KSnk= github.com/dop251/goja_nodejs v0.0.0-20210225215109-d91c329300e7/go.mod h1:hn7BA7c8pLvoGndExHudxTDKZ84Pyvv+90pbBjbTz0Y= github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo= github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= @@ -147,15 +146,17 @@ github.com/eclipse/paho.mqtt.golang v1.2.0/go.mod h1:H9keYFcgq3Qr5OUJm/JZI/i6U7j github.com/edsrzf/mmap-go v1.0.0/go.mod h1:YO35OhQPt3KJa3ryjFM5Bs14WD66h8eGKpfaBNrHW5M= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= -github.com/ethereum/go-ethereum v1.10.17 h1:XEcumY+qSr1cZQaWsQs5Kck3FHB0V2RiMHPdTBJ+oT8= -github.com/ethereum/go-ethereum v1.10.17/go.mod h1:Lt5WzjM07XlXc95YzrhosmR4J9Ahd6X2wyEV2SvGhk0= +github.com/ethereum/go-ethereum v1.10.19 h1:EOR5JbL4MD5yeOqv8W2iC1s4NximrTjqFccUz8lyBRA= +github.com/ethereum/go-ethereum v1.10.19/go.mod h1:IJBNMtzKcNHPtllYihy6BL2IgK1u+32JriaTbdt4v+w= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/fatih/color v1.13.0 h1:8LOYc1KYPPmyKMuN8QV2DNRWNbLo6LZ0iLs8+mlH53w= github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk= +github.com/fjl/gencodec v0.0.0-20220412091415-8bb9e558978c/go.mod h1:AzA8Lj6YtixmJWL+wkKoBGsLWy9gFrAzi4g+5bCKwpY= github.com/fjl/memsize v0.0.0-20190710130421-bcb5799ab5e5/go.mod h1:VvhXpOYNQvB+uIk2RvXzuaQtkQJzzIx6lSBe1xv7hi0= github.com/fogleman/gg v1.2.1-0.20190220221249-0403632d5b90/go.mod h1:R/bRT+9gY/C5z7JzPU0zXsXHKM4/ayA+zqcVNZzPa1k= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= +github.com/garslo/gogen v0.0.0-20170306192744-1d203ffc1f61/go.mod h1:Q0X6pkwTILDlzrGEckF6HKjXe48EgsY/l7K7vhY4MW8= github.com/gballet/go-libpcsclite v0.0.0-20190607065134-2772fd86a8ff/go.mod h1:x7DCsMOv1taUwEWCzT4cmDeAkigA5/QCwUodaVOe8Ww= github.com/getkin/kin-openapi v0.53.0/go.mod h1:7Yn5whZr5kJi6t+kShccXS8ae1APpYTW6yheSwk8Yi4= github.com/getkin/kin-openapi v0.61.0/go.mod h1:7Yn5whZr5kJi6t+kShccXS8ae1APpYTW6yheSwk8Yi4= @@ -242,7 +243,7 @@ github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T github.com/holiman/bloomfilter/v2 v2.0.3/go.mod h1:zpoh+gs7qcpqrHr3dB55AMiJwo0iURXE7ZOP9L9hSkA= github.com/holiman/uint256 v1.2.0/go.mod h1:y4ga/t+u+Xwd7CpDgZESaRcWy0I7XMlTMA25ApIH5Jw= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= -github.com/huin/goupnp v1.0.3-0.20220313090229-ca81a64b4204/go.mod h1:ZxNlw5WqJj6wSsRK5+YfflQGXYfccj5VgQsMNixHM7Y= +github.com/huin/goupnp v1.0.3/go.mod h1:ZxNlw5WqJj6wSsRK5+YfflQGXYfccj5VgQsMNixHM7Y= github.com/huin/goutil v0.0.0-20170803182201-1ca381bf3150/go.mod h1:PpLOETDnJ0o3iZrZfqZzyLl6l7F3c6L1oWn7OICBi6o= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM= @@ -292,6 +293,7 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/kylelemons/godebug v0.0.0-20170224010052-a616ab194758/go.mod h1:B69LEHPfb2qLo0BaaOLcbitczOKLWTsrBG9LczfCD4k= github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= github.com/labstack/echo/v4 v4.2.1/go.mod h1:AA49e0DZ8kk5jTOOCKNuPR6oTnBS0dYiM4FW1e6jwpg= github.com/labstack/gommon v0.3.0/go.mod h1:MULnywXg0yavhxWKc+lOruYdAhDwPK9wf0OL7NoOu+k= @@ -345,6 +347,7 @@ github.com/olekukonko/tablewriter v0.0.5 h1:P2Ga83D34wi1o9J6Wh1mRuqd4mF/x/lgBS7N github.com/olekukonko/tablewriter v0.0.5/go.mod h1:hPp6KlRPjbx+hW8ykQs1w3UBbZlj6HuIJcUGPhkA7kY= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.10.3/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= github.com/onsi/ginkgo v1.14.0/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY= github.com/onsi/gomega v1.4.1/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= @@ -422,10 +425,11 @@ github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.2 h1:4jaiDzPyXQvSd7D0EjG45355tLlV3VOECpq10pLC+8s= github.com/stretchr/testify v1.7.2/go.mod h1:R6va5+xMeoiuVRoj+gSkQ7d3FALtqAAGI1FQKckRals= +github.com/supranational/blst v0.3.8-0.20220526154634-513d2456b344/go.mod h1:jZJtfjgudtNl4en1tzwPIV3KjUnQUvG3/j+w+fVonLw= github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7/go.mod h1:q4W45IWZaF22tdD+VEXcAWRA037jwmWEB5VWYORlTpc= github.com/tidwall/gjson v1.12.1/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= -github.com/tidwall/gjson v1.14.0 h1:6aeJ0bzojgWLa82gDQHcx3S0Lr/O51I9bJ5nv6JFx5w= -github.com/tidwall/gjson v1.14.0/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= +github.com/tidwall/gjson v1.14.1 h1:iymTbGkQBhveq21bEvAQ81I0LEBork8BFe1CUZXdyuo= +github.com/tidwall/gjson v1.14.1/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA= github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM= github.com/tidwall/pretty v1.2.0 h1:RWIZEg2iJ8/g6fDDYzMpobmaoGh5OLl4AXtGUGPcqCs= @@ -452,6 +456,7 @@ github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1: github.com/ybbus/jsonrpc v2.1.2+incompatible/go.mod h1:XJrh1eMSzdIYFbM08flv0wp5G35eRniyeGut1z+LSiE= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= +github.com/yuin/goldmark v1.4.1/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= @@ -481,8 +486,9 @@ golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPh golang.org/x/crypto v0.0.0-20200820211705-5c72a883971a/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20201221181555-eec23a3978ad/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= -golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97 h1:/UOmuWzQfxxo9UtlXMwuQU8CMgg1eZXqTRwkSQJWKOI= golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.0.0-20210921155107-089bfa567519 h1:7I4JAnoQBe7ZtJcBaYHi5UtiO8tQHbUSXxL+pnGRANg= +golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20180807140117-3d87b88a115f/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= @@ -493,6 +499,7 @@ golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm0 golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY= golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= +golang.org/x/exp v0.0.0-20220426173459-3bcf042a4bf5/go.mod h1:lgLbSvA5ygNOMpwM/9anMpWVlVJ7Z+cHWq/eFuinpGE= golang.org/x/image v0.0.0-20180708004352-c73c2afc3b81/go.mod h1:ux5Hcp/YLpHSI86hEcLt0YII63i6oz57MZXIpbrjZUs= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= @@ -511,6 +518,8 @@ golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.5.1/go.mod h1:5OXOZSfqPIIbmVBIIKWRFfZjPR0E5r58TLhUjH0a2Ro= +golang.org/x/mod v0.6.0-dev.0.20211013180041-c96bc1413d57/go.mod h1:3p9vT2HGsQu2K1YbXdKPJLVgG5VJdoTa1poYQBtP1AY= golang.org/x/net v0.0.0-20180719180050-a680a1efc54d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -597,7 +606,6 @@ golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210816183151-1e6c022a8912/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211019181941-9d821ace8654/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211205182925-97ca703d548d h1:FjkYO/PPp4Wi0EAUOVLxePm7qVW4r4ctbWpURyuOD0E= @@ -639,11 +647,13 @@ golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtn golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191126055441-b0650ceb63d9/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191216173652-a0e659d51361/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200108203644-89082a384178/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/tools v0.1.8-0.20211029000441-d6a9af8af023/go.mod h1:nABZi5QlRsZVlzPpHl034qft6wpY4eDcsTt5AaioBiU= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -710,7 +720,6 @@ gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EV gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/natefinch/npipe.v2 v2.0.0-20160621034901-c1b8fa8bdcce/go.mod h1:5AcXVHNjg+BDxry382+8OKon8SEWiKktQR07RKPsv1c= -gopkg.in/olebedev/go-duktape.v3 v3.0.0-20200619000410-60c24ae608a6/go.mod h1:uAJfkITjFhyEEuUfm7bsmCZRbW5WRq8s9EY8HZ6hCns= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= gopkg.in/urfave/cli.v1 v1.20.0/go.mod h1:vuBzUtMdQeixQj8LVd+/98pzhxNGQoyuPBlsXHOQNO0= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= @@ -725,7 +734,6 @@ gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= From d5e858c39ba40f770ebaf0db05f41f895124a40a Mon Sep 17 00:00:00 2001 From: shiatcb Date: Tue, 12 Jul 2022 12:08:15 -0400 Subject: [PATCH 333/376] ROSE-449: segregate coinbase spec requirements verification in check:spec --- cmd/check_spec.go | 310 ++++++++++++++++++++++++---------------- cmd/check_spec_utils.go | 52 +++++-- cmd/root.go | 12 +- 3 files changed, 235 insertions(+), 139 deletions(-) diff --git a/cmd/check_spec.go b/cmd/check_spec.go index fdcda28b..206f1cf1 100644 --- a/cmd/check_spec.go +++ b/cmd/check_spec.go @@ -28,10 +28,27 @@ import ( var ( checkSpecCmd = &cobra.Command{ Use: "check:spec", - Short: "Check a Rosetta implementation satisfies Rosetta spec", - Long: `Detailed Rosetta spec can be found in https://www.rosetta-api.org/docs/Reference.html. - Specifically, check:spec will examine the response from all data and construction API endpoints, - and verifiy they have required fields and the values are properly populated and formatted.`, + Short: "Check that a Rosetta implementation satisfies Rosetta spec", + Long: `Check:spec checks whether a Rosetta implementation satisfies either Coinbase-specific requirements or +minimum requirements specified in rosetta-api.org. + +By default, check:spec will verify only Coinbase spec requirements. To verifiy the minimum requirements as well, +add the --all flag to the check:spec command: + +rosetta-cli check:spec --all --configuration-file [filepath] + +The minimum requirements verify whether an API response contains the required fields, and that the fields are +correctly formatted with proper values. For example, it would check whether the response of /network/list +contains a list of network identifiers. + +The Coinbase specific requirements are not documented in rosetta-api.org. However, we highly recommend that your +implementation satisfies them. This ensures that, when you want to integrate your asset into the Coinbase platform, +you can limit or eliminate implementation issues. + +Here are a few examples of Coinbase spec requirements: +1. The network_identifier in Rosetta configuration should be static. Network upgrade shouldn't change its value. +2. When block_identifier is not specified, the call to /block endpoint should return the tip block. +3. The online_url and offline_url should be different.`, RunE: runCheckSpecCmd, } ) @@ -93,27 +110,37 @@ func newCheckSpec(ctx context.Context) (*checkSpec, error) { } func (cs *checkSpec) networkOptions(ctx context.Context) checkSpecOutput { - printInfo("validating /network/options ...\n") - output := checkSpecOutput{ - api: networkOptions, - validation: map[checkSpecRequirement]checkSpecStatus{ - version: checkSpecSuccess, - allow: checkSpecSuccess, - offlineMode: checkSpecSuccess, - }, - } - defer printInfo("/network/options validated\n") + if checkAllSpecs { + printInfo("validating /network/options ...\n") + output := checkSpecOutput{ + api: networkOptions, + validation: map[checkSpecRequirement]checkSpecStatus{ + version: { + status: checkSpecSuccess, + }, + allow: { + status: checkSpecSuccess, + }, + offlineMode: { + status: checkSpecSuccess, + }, + }, + } + defer printInfo("/network/options validated\n") + + // NetworkOptionsRetry handles validation of /network/options response + // This is an endpoint for offline mode + _, err := cs.offlineFetcher.NetworkOptionsRetry(ctx, Config.Network, nil) + if err != nil { + printError("%v: unable to fetch network options\n", err.Err) + markAllValidationsFailed(output) + return output + } - // NetworkOptionsRetry handles validation of /network/options response - // This is an endpoint for offline mode - _, err := cs.offlineFetcher.NetworkOptionsRetry(ctx, Config.Network, nil) - if err != nil { - printError("%v: unable to fetch network options\n", err.Err) - markAllValidationsFailed(output) return output } - return output + return checkSpecOutput{} } func (cs *checkSpec) networkList(ctx context.Context) checkSpecOutput { @@ -121,26 +148,38 @@ func (cs *checkSpec) networkList(ctx context.Context) checkSpecOutput { output := checkSpecOutput{ api: networkList, validation: map[checkSpecRequirement]checkSpecStatus{ - networkIDs: checkSpecSuccess, - offlineMode: checkSpecSuccess, - staticNetworkID: checkSpecSuccess, + staticNetworkID: { + status: checkSpecSuccess, + coinbaseSpec: true, + }, }, } + + if checkAllSpecs { + output.validation[networkIDs] = checkSpecStatus{ + status: checkSpecSuccess, + } + output.validation[offlineMode] = checkSpecStatus{ + status: checkSpecSuccess, + } + } + defer printInfo("/network/list validated\n") + networks, err := cs.offlineFetcher.NetworkListRetry(ctx, nil) // endpoint for offline mode - networks, err := cs.offlineFetcher.NetworkListRetry(ctx, nil) if err != nil { printError("%v: unable to fetch network list", err.Err) markAllValidationsFailed(output) return output } - if len(networks.NetworkIdentifiers) == 0 { + if checkAllSpecs && len(networks.NetworkIdentifiers) == 0 { printError("network_identifiers is required") setValidationStatusFailed(output, networkIDs) } + // static network ID for _, network := range networks.NetworkIdentifiers { if isEqual(network.Network, Config.Network.Network) && isEqual(network.Blockchain, Config.Network.Blockchain) { @@ -148,50 +187,57 @@ func (cs *checkSpec) networkList(ctx context.Context) checkSpecOutput { } } - // static network ID printError("network_identifier in configuration file is not returned by /network/list") setValidationStatusFailed(output, staticNetworkID) return output } func (cs *checkSpec) accountCoins(ctx context.Context) checkSpecOutput { - printInfo("validating /account/coins ...\n") - output := checkSpecOutput{ - api: accountCoins, - validation: map[checkSpecRequirement]checkSpecStatus{ - blockID: checkSpecSuccess, - coins: checkSpecSuccess, - }, - } - defer printInfo("/account/coins validated\n") - - if isUTXO() { - acct, _, currencies, err := cs.getAccount(ctx) - if err != nil { - printError("%v: unable to get an account\n", err) - markAllValidationsFailed(output) - return output - } - if err != nil { - printError("%v\n", errAccountNullPointer) - markAllValidationsFailed(output) - return output + if checkAllSpecs { + printInfo("validating /account/coins ...\n") + output := checkSpecOutput{ + api: accountCoins, + validation: map[checkSpecRequirement]checkSpecStatus{ + blockID: { + status: checkSpecSuccess, + }, + coins: { + status: checkSpecSuccess, + }, + }, } + defer printInfo("/account/coins validated\n") + + if isUTXO() { + acct, _, currencies, err := cs.getAccount(ctx) + if err != nil { + printError("%v: unable to get an account\n", err) + markAllValidationsFailed(output) + return output + } + if err != nil { + printError("%v\n", errAccountNullPointer) + markAllValidationsFailed(output) + return output + } - _, _, _, fetchErr := cs.onlineFetcher.AccountCoinsRetry( - ctx, - Config.Network, - acct, - false, - currencies) - if fetchErr != nil { - printError("%v: unable to get coins for account: %v\n", fetchErr.Err, *acct) - markAllValidationsFailed(output) - return output + _, _, _, fetchErr := cs.onlineFetcher.AccountCoinsRetry( + ctx, + Config.Network, + acct, + false, + currencies) + if fetchErr != nil { + printError("%v: unable to get coins for account: %v\n", fetchErr.Err, *acct) + markAllValidationsFailed(output) + return output + } } + + return output } - return output + return checkSpecOutput{} } func (cs *checkSpec) block(ctx context.Context) checkSpecOutput { @@ -199,12 +245,20 @@ func (cs *checkSpec) block(ctx context.Context) checkSpecOutput { output := checkSpecOutput{ api: block, validation: map[checkSpecRequirement]checkSpecStatus{ - idempotent: checkSpecSuccess, - defaultTip: checkSpecSuccess, + defaultTip: { + status: checkSpecSuccess, + coinbaseSpec: true, + }, }, } defer printInfo("/block validated\n") + if checkAllSpecs { + output.validation[idempotent] = checkSpecStatus{ + status: checkSpecSuccess, + } + } + res, fetchErr := cs.onlineFetcher.NetworkStatusRetry(ctx, Config.Network, nil) if fetchErr != nil { printError("%v: unable to get network status\n", fetchErr.Err) @@ -212,27 +266,29 @@ func (cs *checkSpec) block(ctx context.Context) checkSpecOutput { return output } - // multiple calls with the same hash should return the same block - var block *types.Block - tip := res.CurrentBlockIdentifier - callTimes := 3 + if checkAllSpecs { + // multiple calls with the same hash should return the same block + var block *types.Block + tip := res.CurrentBlockIdentifier + callTimes := 3 - for i := 0; i < callTimes; i++ { - blockID := types.PartialBlockIdentifier{ - Hash: &tip.Hash, - } - b, fetchErr := cs.onlineFetcher.BlockRetry(ctx, Config.Network, &blockID) - if fetchErr != nil { - printError("%v: unable to fetch block %v\n", fetchErr.Err, blockID) - markAllValidationsFailed(output) - return output - } + for i := 0; i < callTimes; i++ { + blockID := types.PartialBlockIdentifier{ + Hash: &tip.Hash, + } + b, fetchErr := cs.onlineFetcher.BlockRetry(ctx, Config.Network, &blockID) + if fetchErr != nil { + printError("%v: unable to fetch block %v\n", fetchErr.Err, blockID) + markAllValidationsFailed(output) + return output + } - if block == nil { - block = b - } else if !isEqual(types.Hash(*block), types.Hash(*b)) { - printError("%v\n", errBlockNotIdempotent) - setValidationStatusFailed(output, idempotent) + if block == nil { + block = b + } else if !isEqual(types.Hash(*block), types.Hash(*b)) { + printError("%v\n", errBlockNotIdempotent) + setValidationStatusFailed(output, idempotent) + } } } @@ -243,11 +299,11 @@ func (cs *checkSpec) block(ctx context.Context) checkSpecOutput { setValidationStatusFailed(output, defaultTip) return output } - tip = res.CurrentBlockIdentifier + tip := res.CurrentBlockIdentifier // tip shoud be returned if block_identifier is not specified emptyBlockID := &types.PartialBlockIdentifier{} - block, fetchErr = cs.onlineFetcher.BlockRetry(ctx, Config.Network, emptyBlockID) + block, fetchErr := cs.onlineFetcher.BlockRetry(ctx, Config.Network, emptyBlockID) if fetchErr != nil { printError("%v: unable to fetch tip block\n", fetchErr.Err) setValidationStatusFailed(output, defaultTip) @@ -264,51 +320,59 @@ func (cs *checkSpec) block(ctx context.Context) checkSpecOutput { } func (cs *checkSpec) errorObject(ctx context.Context) checkSpecOutput { - printInfo("validating error object ...\n") - output := checkSpecOutput{ - api: errorObject, - validation: map[checkSpecRequirement]checkSpecStatus{ - errorCode: checkSpecSuccess, - errorMessage: checkSpecSuccess, - }, - } - defer printInfo("error object validated\n") - - printInfo("%v\n", "sending request to /network/status ...") - emptyNetwork := &types.NetworkIdentifier{} - _, err := cs.onlineFetcher.NetworkStatusRetry(ctx, emptyNetwork, nil) - validateErrorObject(err, output) - - printInfo("%v\n", "sending request to /network/options ...") - _, err = cs.onlineFetcher.NetworkOptionsRetry(ctx, emptyNetwork, nil) - validateErrorObject(err, output) - - printInfo("%v\n", "sending request to /account/balance ...") - emptyAcct := &types.AccountIdentifier{} - emptyPartBlock := &types.PartialBlockIdentifier{} - emptyCur := []*types.Currency{} - _, _, _, err = cs.onlineFetcher.AccountBalanceRetry(ctx, emptyNetwork, emptyAcct, emptyPartBlock, emptyCur) - validateErrorObject(err, output) - - if isUTXO() { - printInfo("%v\n", "sending request to /account/coins ...") - _, _, _, err = cs.onlineFetcher.AccountCoinsRetry(ctx, emptyNetwork, emptyAcct, false, emptyCur) + if checkAllSpecs { + printInfo("validating error object ...\n") + output := checkSpecOutput{ + api: errorObject, + validation: map[checkSpecRequirement]checkSpecStatus{ + errorCode: { + status: checkSpecSuccess, + }, + errorMessage: { + status: checkSpecSuccess, + }, + }, + } + defer printInfo("error object validated\n") + + printInfo("%v\n", "sending request to /network/status ...") + emptyNetwork := &types.NetworkIdentifier{} + _, err := cs.onlineFetcher.NetworkStatusRetry(ctx, emptyNetwork, nil) validateErrorObject(err, output) - } else { - printInfo("%v\n", "skip /account/coins for account based chain") - } - printInfo("%v\n", "sending request to /block ...") - _, err = cs.onlineFetcher.BlockRetry(ctx, emptyNetwork, emptyPartBlock) - validateErrorObject(err, output) + printInfo("%v\n", "sending request to /network/options ...") + _, err = cs.onlineFetcher.NetworkOptionsRetry(ctx, emptyNetwork, nil) + validateErrorObject(err, output) - printInfo("%v\n", "sending request to /block/transaction ...") - emptyTx := []*types.TransactionIdentifier{} - emptyBlock := &types.BlockIdentifier{} - _, err = cs.onlineFetcher.UnsafeTransactions(ctx, emptyNetwork, emptyBlock, emptyTx) - validateErrorObject(err, output) + printInfo("%v\n", "sending request to /account/balance ...") + emptyAcct := &types.AccountIdentifier{} + emptyPartBlock := &types.PartialBlockIdentifier{} + emptyCur := []*types.Currency{} + _, _, _, err = cs.onlineFetcher.AccountBalanceRetry(ctx, emptyNetwork, emptyAcct, emptyPartBlock, emptyCur) + validateErrorObject(err, output) - return output + if isUTXO() { + printInfo("%v\n", "sending request to /account/coins ...") + _, _, _, err = cs.onlineFetcher.AccountCoinsRetry(ctx, emptyNetwork, emptyAcct, false, emptyCur) + validateErrorObject(err, output) + } else { + printInfo("%v\n", "skip /account/coins for account based chain") + } + + printInfo("%v\n", "sending request to /block ...") + _, err = cs.onlineFetcher.BlockRetry(ctx, emptyNetwork, emptyPartBlock) + validateErrorObject(err, output) + + printInfo("%v\n", "sending request to /block/transaction ...") + emptyTx := []*types.TransactionIdentifier{} + emptyBlock := &types.BlockIdentifier{} + _, err = cs.onlineFetcher.UnsafeTransactions(ctx, emptyNetwork, emptyBlock, emptyTx) + validateErrorObject(err, output) + + return output + } + + return checkSpecOutput{} } // Searching for an account backwards from the tip diff --git a/cmd/check_spec_utils.go b/cmd/check_spec_utils.go index ebbb26cc..ab70d3f2 100644 --- a/cmd/check_spec_utils.go +++ b/cmd/check_spec_utils.go @@ -17,6 +17,7 @@ package cmd import ( "errors" "fmt" + "strconv" "github.com/coinbase/rosetta-sdk-go/fetcher" "github.com/fatih/color" @@ -33,7 +34,6 @@ var ( type checkSpecAPI string type checkSpecRequirement string -type checkSpecStatus string const ( networkList checkSpecAPI = "/network/list" @@ -58,10 +58,16 @@ const ( errorMessage checkSpecRequirement = "error message is required" diffURLs checkSpecRequirement = "offline_url should be different from offline_url and not empty" - checkSpecSuccess checkSpecStatus = "Success" - checkSpecFailure checkSpecStatus = "Failure" + checkSpecSuccess string = "Success" + checkSpecFailure string = "Failure" + cbSpec bool = true ) +type checkSpecStatus struct { + status string + coinbaseSpec bool +} + type checkSpecOutput struct { api checkSpecAPI validation map[checkSpecRequirement]checkSpecStatus @@ -71,7 +77,10 @@ func twoModes() checkSpecOutput { output := checkSpecOutput{ api: modes, validation: map[checkSpecRequirement]checkSpecStatus{ - diffURLs: checkSpecSuccess, + diffURLs: { + status: checkSpecSuccess, + coinbaseSpec: cbSpec, + }, }, } @@ -85,13 +94,19 @@ func twoModes() checkSpecOutput { } func markAllValidationsFailed(output checkSpecOutput) { - for k := range output.validation { - output.validation[k] = checkSpecFailure + for k, v := range output.validation { + output.validation[k] = checkSpecStatus{ + status: checkSpecFailure, + coinbaseSpec: v.coinbaseSpec, + } } } func setValidationStatusFailed(output checkSpecOutput, req checkSpecRequirement) { - output.validation[req] = checkSpecFailure + output.validation[req] = checkSpecStatus{ + status: checkSpecFailure, + coinbaseSpec: output.validation[req].coinbaseSpec, + } } func validateErrorObject(err *fetcher.Error, output checkSpecOutput) { @@ -120,8 +135,8 @@ func printSuccess(format string, a ...interface{}) { fmt.Print(color.GreenString(format, a...)) } -func printValidationResult(format string, status checkSpecStatus, a ...interface{}) { - if status == checkSpecFailure { +func printValidationResult(format string, css checkSpecStatus, a ...interface{}) { + if css.status == checkSpecFailure { printError(format, a...) } else { printSuccess(format, a...) @@ -129,9 +144,9 @@ func printValidationResult(format string, status checkSpecStatus, a ...interface } func printCheckSpecOutputHeader() { - printInfo("%v\n", "+--------------------------+-------------------------------------------------------------------+-----------+") - printInfo("%v\n", "| API | Requirement | Status |") - printInfo("%v\n", "+--------------------------+-------------------------------------------------------------------+-----------+") + printInfo("%v\n", "+--------------------------+-------------------------------------------------------------------+-----------+-----------------+") + printInfo("%v\n", "| API | Requirement | Status | Coinbase Spec |") + printInfo("%v\n", "+--------------------------+-------------------------------------------------------------------+-----------+-----------------+") } func printCheckSpecOutputBody(output checkSpecOutput) { @@ -152,12 +167,19 @@ func printCheckSpecOutputBody(output checkSpecOutput) { // print validation status printInfo("%v", "| ") - printValidationResult("%v", v, v) - for j := 0; j < 9-len(v); j++ { + printValidationResult("%v", v, v.status) + for j := 0; j < 9-len(v.status); j++ { + printInfo("%v", " ") + } + + // print coinbase spec flag + printInfo("%v", "| ") + printValidationResult("%v", v, v.coinbaseSpec) + for j := 0; j < 11-len(strconv.FormatBool(v.coinbaseSpec)); j++ { printInfo("%v", " ") } printInfo("%v\n", "|") - printInfo("%v\n", "+--------------------------+-------------------------------------------------------------------+-----------+") + printInfo("%v\n", "+--------------------------+-------------------------------------------------------------------+-----------+-----------------+") } } diff --git a/cmd/root.go b/cmd/root.go index c7d05643..45984c19 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -80,6 +80,10 @@ var ( // logged to the console. OnlyChanges bool + // allSpecs is a boolean indicating whether check:spec should verify only Coinbase + // spec requirements, or the minimum requirements as well. + checkAllSpecs bool + // If non-empty, used to validate that /network/options matches the contents of the file // located at this path. The intended use case is someone previously ran // utils:asserter-configuration `asserterConfigurationFile`, so the validation is being done @@ -308,6 +312,12 @@ default values.`, rootCmd.AddCommand(checkPerfCmd) // check:spec + checkSpecCmd.Flags().BoolVar( + &checkAllSpecs, + "all", + false, + `Verify both minimum and Coinbase spec requirements`, + ) rootCmd.AddCommand(checkSpecCmd) } @@ -397,6 +407,6 @@ var versionCmd = &cobra.Command{ Use: "version", Short: "Print rosetta-cli version", Run: func(cmd *cobra.Command, args []string) { - fmt.Println("v0.7.9") + fmt.Println("v0.8.0") }, } From f1972db61fcba49352c62713a8e522ea14b0ee28 Mon Sep 17 00:00:00 2001 From: Jingfu Wang Date: Wed, 13 Jul 2022 18:31:26 -0400 Subject: [PATCH 334/376] feat: update dockerfile version Signed-off-by: Jingfu Wang --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 4840968b..2e184edc 100644 --- a/Dockerfile +++ b/Dockerfile @@ -31,7 +31,7 @@ RUN mkdir -p "$GOPATH/src" "$GOPATH/bin" && chmod -R 777 "$GOPATH" WORKDIR /go/src -ARG VERSION=v0.7.9 +ARG VERSION=v0.8.0 RUN git clone https://github.com/coinbase/rosetta-cli.git && \ cd rosetta-cli && \ git fetch --all --tags && \ From ef16096040eec9d926e4a9ce99c2b220f37edc04 Mon Sep 17 00:00:00 2001 From: Jingfu Wang Date: Thu, 14 Jul 2022 15:57:39 -0400 Subject: [PATCH 335/376] feat: update compile script and instruction Signed-off-by: Jingfu Wang --- README.md | 13 +++++++------ scripts/compile.sh | 2 +- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index 1c3d68f8..3451937d 100644 --- a/README.md +++ b/README.md @@ -89,14 +89,15 @@ While working on improvements to this repository, we recommend that you use thes * `make release` to run one last check before opening a PR * `make compile version=RELEASE_TAG` to generate binaries -If you are developing on both the `rosetta-cli` and `rosetta-sdk-go` repositories, use [go.mod replace](https://golang.org/ref/mod#go-mod-file-replace) to reference local changes: +If you are developing on both the `rosetta-cli` and `rosetta-sdk-go` repositories, use [go.mod replace](https://golang.org/ref/mod#go-mod-file-replace) to reference local changes: ``` -replace "github.com/coinbase/rosetta-sdk-go" v0.6.8 => "../rosetta-sdk-go" +replace "github.com/coinbase/rosetta-sdk-go" v0.6.8 => "" ``` -## Release -* When we release a new rosetta-cli version, please update the [version number](https://github.com/coinbase/rosetta-cli/blob/master/cmd/root.go#L297) so that `rosetta-cli version` command can print the correct value. -* Create binaries and upload all the binaries in the new release tag (Eg: https://github.com/coinbase/rosetta-cli/releases/tag/v0.7.7) - * `make compile version=` +### Release +* When we release a new rosetta-cli version, please update the version number to follow [PR](https://github.com/coinbase/rosetta-cli/pull/334) so that `rosetta-cli version` command can print the correct value. +* Create binaries and upload all the binaries in the new release tag (e.g. https://github.com/coinbase/rosetta-cli/releases/tag/v0.7.7) + * Ensure `$GOPATH/bin` is added to `$PATH` + * Run `make compile version=` * All the binaries will be created in the `bin` folder and should have extension as `tar.gz` and new version number ### Helper/Handler diff --git a/scripts/compile.sh b/scripts/compile.sh index 8ce67c52..764b9fbb 100755 --- a/scripts/compile.sh +++ b/scripts/compile.sh @@ -16,7 +16,7 @@ VERSION=$1; -go get github.com/crazy-max/xgo +go install github.com/crazy-max/xgo@latest MAC_TARGETS="darwin/amd64,darwin/arm64" LINUX_TARGETS="linux/amd64,linux/arm64,linux/mips64,linux/mips64le,linux/ppc64le,linux/s390x" From fdd7178dfdb4eb3d5a8080cd925b5a637f434f6b Mon Sep 17 00:00:00 2001 From: jingwei zhuang Date: Mon, 8 Aug 2022 17:51:29 -0700 Subject: [PATCH 336/376] enable allinmemory in cli --- configuration/types.go | 15 +++++++++++++ go.mod | 2 ++ pkg/tester/data.go | 51 +++++++++++++++++++++++++++++++++++++----- 3 files changed, 62 insertions(+), 6 deletions(-) diff --git a/configuration/types.go b/configuration/types.go index 603afbef..3f3af83f 100644 --- a/configuration/types.go +++ b/configuration/types.go @@ -414,6 +414,21 @@ type Configuration struct { // but can use 10s of GBs of RAM, even with pruning enabled. MemoryLimitDisabled bool `json:"memory_limit_disabled"` + // AllInMemoryDisabled configures storage to increase memory + // usage. Enabling this massively increases performance + // but can use 20s of GBs of RAM, even with pruning enabled. + AllInMemoryEnabled bool `json:"all_in_memory_enabled"` + + // TableSize unit is GB, enable users to define MaxTableSize + // when AllInMemoryEnabled == true, Cli will look up this config + // default value is 6, modification range is [3, 100] + TableSize *int64 `json:"table_size,omitempty"` + + // TableSize unit is MB, enable users to define ValueLogFileSize + // when AllInMemoryEnabled == true, Cli will look up this config + // default value is 512, modification range is [256, 2048] + ValueLogFileSize *int64 `json:"value_log_file_size,omitempty"` + // SeenBlockWorkers is the number of goroutines spawned to store // seen blocks in storage before we attempt to sequence. If not populated, // this value defaults to runtime.NumCPU(). diff --git a/go.mod b/go.mod index 4db9cbf4..ac5cbc7e 100644 --- a/go.mod +++ b/go.mod @@ -2,6 +2,8 @@ module github.com/coinbase/rosetta-cli go 1.16 +replace github.com/coinbase/rosetta-sdk-go => /Users/jingweizhuang/crypto/rosetta-sdk-go + require ( github.com/coinbase/rosetta-sdk-go v0.7.11-0.20220629212620-136b591fb3f4 github.com/fatih/color v1.13.0 diff --git a/pkg/tester/data.go b/pkg/tester/data.go index e74e9bdc..23662230 100644 --- a/pkg/tester/data.go +++ b/pkg/tester/data.go @@ -65,6 +65,18 @@ const ( // EndAtTipCheckInterval is the frequency that EndAtTip condition // is evaludated EndAtTipCheckInterval = 10 * time.Second + + //MinTableSize unit is GB + MinTableSize = int64(3) + + //MaxTableSize unit is GB + MaxTableSize = int64(100) + + //MinTableSize unit is MB + MinValueLogFileSize = int64(256) + + //MaxTableSize unit is MB + MaxValueLogFileSize = int64(2048) ) var _ http.Handler = (*DataTester)(nil) @@ -152,17 +164,44 @@ func InitializeData( } opts := []database.BadgerOption{} - if config.CompressionDisabled { - opts = append(opts, database.WithoutCompression()) - } - if config.MemoryLimitDisabled { + dataPathBackup := dataPath + + if config.AllInMemoryEnabled{ opts = append( opts, - database.WithCustomSettings(database.PerformanceBadgerOptions(dataPath)), + database.WithCustomSettings(database.AllInMemoryBadgerOptions(dataPath)), + database.WithoutCompression(), ) + if(config.TableSize != nil) { + if(*config.TableSize >= MinTableSize && *config.TableSize <= MaxTableSize) { + opts = append( + opts, + database.WithTableSize(*config.TableSize), + ) + } + } + if(config.ValueLogFileSize != nil) { + if(*config.TableSize >= MinValueLogFileSize && *config.TableSize <= MinValueLogFileSize) { + opts = append( + opts, + database.WithValueLogFileSize(*config.TableSize), + ) + } + } + dataPathBackup = "" + } else { + if config.CompressionDisabled { + opts = append(opts, database.WithoutCompression()) + } + if config.MemoryLimitDisabled { + opts = append( + opts, + database.WithCustomSettings(database.PerformanceBadgerOptions(dataPath)), + ) + } } - localStore, err := database.NewBadgerDatabase(ctx, dataPath, opts...) + localStore, err := database.NewBadgerDatabase(ctx, dataPathBackup, opts...) if err != nil { return nil, fmt.Errorf("%s: unable to initialize database", err.Error()) } From 71d29934a8e357d3158bcd4d6d867008878af1d1 Mon Sep 17 00:00:00 2001 From: jingwei zhuang Date: Mon, 8 Aug 2022 20:37:05 -0700 Subject: [PATCH 337/376] updates --- examples/configuration/default.json | 1 + 1 file changed, 1 insertion(+) diff --git a/examples/configuration/default.json b/examples/configuration/default.json index df5c68ab..c2ca45a7 100644 --- a/examples/configuration/default.json +++ b/examples/configuration/default.json @@ -15,6 +15,7 @@ "log_configuration": false, "compression_disabled": false, "memory_limit_disabled": false, + "all_in_memory_enabled": false, "error_stack_trace_disabled": false, "coin_supported": false, "construction": null, From 0e53ff336041b1dd68a9175e632a8dba04083cef Mon Sep 17 00:00:00 2001 From: jingwei zhuang Date: Fri, 12 Aug 2022 13:30:01 -0700 Subject: [PATCH 338/376] updates --- cmd/root.go | 26 +++++++++++++++++++++++++- configuration/types.go | 4 ++-- go.mod | 4 +--- go.sum | 4 ++-- pkg/tester/data.go | 29 ++++++++++++++++------------- 5 files changed, 46 insertions(+), 21 deletions(-) diff --git a/cmd/root.go b/cmd/root.go index 45984c19..a0fcc0fd 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -55,7 +55,8 @@ var ( dataResultFile string constructionResultFile string dataDirectory string - + allInMemory bool + tableSize int64 // Config is the populated *configuration.Configuration from // the configurationFile. If none is provided, this is set // to the default settings. @@ -263,6 +264,21 @@ default values.`, "", "Data-dir configures the location of logs and data for validation. This will override the data_directory from configuration file", ) + + checkDataCmd.Flags().Int64Var( + &tableSize, + "table_size", + 0, + `Table_size configures the TableSize for badger DB. If tableSize != 0, this will override the table_size from configuration file`, + ) + + checkDataCmd.Flags().BoolVar( + &allInMemory, + "all_in_memory", + false, + "All_in_memory configures badger DB inMeomry option. This will override the all_in_memory_enabled to true from configuration file", + ) + rootCmd.AddCommand(checkDataCmd) checkConstructionCmd.Flags().StringVar( &asserterConfigurationFile, @@ -372,6 +388,14 @@ func initConfig() { if len(dataDirectory) != 0 { Config.DataDirectory = dataDirectory } + + if allInMemory { + Config.AllInMemoryEnabled = allInMemory + } + + if tableSize != 0 { + Config.TableSize = &tableSize + } } func ensureDataDirectoryExists() { diff --git a/configuration/types.go b/configuration/types.go index 3f3af83f..93c8d4db 100644 --- a/configuration/types.go +++ b/configuration/types.go @@ -421,12 +421,12 @@ type Configuration struct { // TableSize unit is GB, enable users to define MaxTableSize // when AllInMemoryEnabled == true, Cli will look up this config - // default value is 6, modification range is [3, 100] + // default value is 6, modification range is [2, 100] TableSize *int64 `json:"table_size,omitempty"` // TableSize unit is MB, enable users to define ValueLogFileSize // when AllInMemoryEnabled == true, Cli will look up this config - // default value is 512, modification range is [256, 2048] + // default value is 512, modification range is [128, 2048] ValueLogFileSize *int64 `json:"value_log_file_size,omitempty"` // SeenBlockWorkers is the number of goroutines spawned to store diff --git a/go.mod b/go.mod index ac5cbc7e..dcb81ec5 100644 --- a/go.mod +++ b/go.mod @@ -2,10 +2,8 @@ module github.com/coinbase/rosetta-cli go 1.16 -replace github.com/coinbase/rosetta-sdk-go => /Users/jingweizhuang/crypto/rosetta-sdk-go - require ( - github.com/coinbase/rosetta-sdk-go v0.7.11-0.20220629212620-136b591fb3f4 + github.com/coinbase/rosetta-sdk-go v0.7.11 github.com/fatih/color v1.13.0 github.com/golang/protobuf v1.5.2 // indirect github.com/google/go-cmp v0.5.6 // indirect diff --git a/go.sum b/go.sum index 8797de37..4db8ed05 100644 --- a/go.sum +++ b/go.sum @@ -97,8 +97,8 @@ github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDk github.com/cloudflare/cloudflare-go v0.14.0/go.mod h1:EnwdgGMaFOruiPZRFSgn+TsQ3hQ7C/YWzIGLeu5c304= github.com/coinbase/kryptology v1.8.0 h1:Aoq4gdTsJhSU3lNWsD5BWmFSz2pE0GlmrljaOxepdYY= github.com/coinbase/kryptology v1.8.0/go.mod h1:RYXOAPdzOGUe3qlSFkMGn58i3xUA8hmxYHksuq+8ciI= -github.com/coinbase/rosetta-sdk-go v0.7.11-0.20220629212620-136b591fb3f4 h1:cHRCEzEk+4xerPtuJgO5a71qp7jIwGDKCKq+pte2VHA= -github.com/coinbase/rosetta-sdk-go v0.7.11-0.20220629212620-136b591fb3f4/go.mod h1:HLTqSTSnOGLWHGTxoUJQO2TLuKkas1B9i/7ByerK6lM= +github.com/coinbase/rosetta-sdk-go v0.7.11 h1:T7camDGstlX2ENVE0QHR1AVu3asSXED0vg1xDaF6bYE= +github.com/coinbase/rosetta-sdk-go v0.7.11/go.mod h1:HLTqSTSnOGLWHGTxoUJQO2TLuKkas1B9i/7ByerK6lM= github.com/consensys/bavard v0.1.8-0.20210406032232-f3452dc9b572/go.mod h1:Bpd0/3mZuaj6Sj+PqrmIquiOKy397AKGThQPaGzNXAQ= github.com/consensys/bavard v0.1.8-0.20210915155054-088da2f7f54a/go.mod h1:9ItSMtA/dXMAiL7BG6bqW2m3NdSEObYWoH223nGHukI= github.com/consensys/gnark-crypto v0.4.1-0.20210426202927-39ac3d4b3f1f/go.mod h1:815PAHg3wvysy0SyIqanF8gZ0Y1wjk/hrDHD/iT88+Q= diff --git a/pkg/tester/data.go b/pkg/tester/data.go index 23662230..42bac274 100644 --- a/pkg/tester/data.go +++ b/pkg/tester/data.go @@ -67,13 +67,13 @@ const ( EndAtTipCheckInterval = 10 * time.Second //MinTableSize unit is GB - MinTableSize = int64(3) + MinTableSize = int64(2) //MaxTableSize unit is GB MaxTableSize = int64(100) //MinTableSize unit is MB - MinValueLogFileSize = int64(256) + MinValueLogFileSize = int64(128) //MaxTableSize unit is MB MaxValueLogFileSize = int64(2048) @@ -172,6 +172,20 @@ func InitializeData( database.WithCustomSettings(database.AllInMemoryBadgerOptions(dataPath)), database.WithoutCompression(), ) + dataPathBackup = "" + } else { + if config.CompressionDisabled { + opts = append(opts, database.WithoutCompression()) + } + if config.MemoryLimitDisabled { + opts = append( + opts, + database.WithCustomSettings(database.PerformanceBadgerOptions(dataPath)), + ) + } + } + + if config.AllInMemoryEnabled || config.MemoryLimitDisabled { if(config.TableSize != nil) { if(*config.TableSize >= MinTableSize && *config.TableSize <= MaxTableSize) { opts = append( @@ -188,17 +202,6 @@ func InitializeData( ) } } - dataPathBackup = "" - } else { - if config.CompressionDisabled { - opts = append(opts, database.WithoutCompression()) - } - if config.MemoryLimitDisabled { - opts = append( - opts, - database.WithCustomSettings(database.PerformanceBadgerOptions(dataPath)), - ) - } } localStore, err := database.NewBadgerDatabase(ctx, dataPathBackup, opts...) From c3a4f4be5f4b91a04bfe39e1de8574d89a57cef7 Mon Sep 17 00:00:00 2001 From: jingwei zhuang Date: Fri, 12 Aug 2022 13:41:51 -0700 Subject: [PATCH 339/376] updates --- cmd/root.go | 1 + configuration/types.go | 6 +++--- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/cmd/root.go b/cmd/root.go index a0fcc0fd..ad48879b 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -57,6 +57,7 @@ var ( dataDirectory string allInMemory bool tableSize int64 + // Config is the populated *configuration.Configuration from // the configurationFile. If none is provided, this is set // to the default settings. diff --git a/configuration/types.go b/configuration/types.go index 93c8d4db..2534cbc3 100644 --- a/configuration/types.go +++ b/configuration/types.go @@ -416,16 +416,16 @@ type Configuration struct { // AllInMemoryDisabled configures storage to increase memory // usage. Enabling this massively increases performance - // but can use 20s of GBs of RAM, even with pruning enabled. + // but can use >20s of GBs of RAM, even with pruning enabled. AllInMemoryEnabled bool `json:"all_in_memory_enabled"` // TableSize unit is GB, enable users to define MaxTableSize - // when AllInMemoryEnabled == true, Cli will look up this config + // when AllInMemoryEnabled == true or MemoryLimitDisabled== true, Cli will look up this config // default value is 6, modification range is [2, 100] TableSize *int64 `json:"table_size,omitempty"` // TableSize unit is MB, enable users to define ValueLogFileSize - // when AllInMemoryEnabled == true, Cli will look up this config + // when AllInMemoryEnabled == true or MemoryLimitDisabled== true, Cli will look up this config // default value is 512, modification range is [128, 2048] ValueLogFileSize *int64 `json:"value_log_file_size,omitempty"` From dccf2c45393327a49a3e7c535547339e0aee3931 Mon Sep 17 00:00:00 2001 From: jingwei zhuang Date: Fri, 12 Aug 2022 14:20:51 -0700 Subject: [PATCH 340/376] updates --- pkg/tester/data.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/pkg/tester/data.go b/pkg/tester/data.go index 42bac274..12ed5a9d 100644 --- a/pkg/tester/data.go +++ b/pkg/tester/data.go @@ -172,6 +172,7 @@ func InitializeData( database.WithCustomSettings(database.AllInMemoryBadgerOptions(dataPath)), database.WithoutCompression(), ) + // for all in memory mode, the path need to be "", as badgerDB will not write to disk dataPathBackup = "" } else { if config.CompressionDisabled { @@ -185,6 +186,10 @@ func InitializeData( } } + // If we enable all-in-memory or L0-in-memory mode, badger DB's TableSize and ValueLogFileSize will change + // according to users config. tableSize means the LSM table size, when the table more than the tableSize, + // will trigger a compact. + // In default mode, we will not change the badger DB's TableSize and ValueLogFileSize for limiting memory usage if config.AllInMemoryEnabled || config.MemoryLimitDisabled { if(config.TableSize != nil) { if(*config.TableSize >= MinTableSize && *config.TableSize <= MaxTableSize) { From aa77764295e50252c20b4e0cbca955c830bdc449 Mon Sep 17 00:00:00 2001 From: jingwei zhuang Date: Fri, 12 Aug 2022 15:59:51 -0700 Subject: [PATCH 341/376] update --- cmd/root.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/cmd/root.go b/cmd/root.go index ad48879b..b362879e 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -268,16 +268,16 @@ default values.`, checkDataCmd.Flags().Int64Var( &tableSize, - "table_size", + "table-size", 0, - `Table_size configures the TableSize for badger DB. If tableSize != 0, this will override the table_size from configuration file`, + `Table-size configures the TableSize for badger DB. If tableSize != 0, this will override the table_size from configuration file`, ) checkDataCmd.Flags().BoolVar( &allInMemory, - "all_in_memory", + "all-in-memory", false, - "All_in_memory configures badger DB inMeomry option. This will override the all_in_memory_enabled to true from configuration file", + "All-in-memory configures badger DB inMeomry option. This will override the all_in_memory_enabled to true from configuration file", ) rootCmd.AddCommand(checkDataCmd) From a6397808eb34229738ae21e51336b6a6191d1a6b Mon Sep 17 00:00:00 2001 From: jingwei zhuang Date: Fri, 12 Aug 2022 22:57:11 -0700 Subject: [PATCH 342/376] update --- cmd/root.go | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/cmd/root.go b/cmd/root.go index b362879e..3e349f4c 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -55,7 +55,7 @@ var ( dataResultFile string constructionResultFile string dataDirectory string - allInMemory bool + turnOnMemoryMode bool tableSize int64 // Config is the populated *configuration.Configuration from @@ -270,14 +270,14 @@ default values.`, &tableSize, "table-size", 0, - `Table-size configures the TableSize for badger DB. If tableSize != 0, this will override the table_size from configuration file`, + `Table-size configures the TableSize for badger DB. If table-size != 0, this will override the table_size from configuration file`, ) checkDataCmd.Flags().BoolVar( - &allInMemory, - "all-in-memory", + &turnOnMemoryMode, + "turn-on-memory-mode", false, - "All-in-memory configures badger DB inMeomry option. This will override the all_in_memory_enabled to true from configuration file", + "turn-on-memory-mode configures badger DB inMeomry option. if turn-on-memory-mode=true This will override the all_in_memory_enabled", ) rootCmd.AddCommand(checkDataCmd) @@ -390,8 +390,8 @@ func initConfig() { Config.DataDirectory = dataDirectory } - if allInMemory { - Config.AllInMemoryEnabled = allInMemory + if turnOnMemoryMode { + Config.AllInMemoryEnabled = turnOnMemoryMode } if tableSize != 0 { From dfa1d2d960f1e044fe1fc4ca572c28e6b02fc0e6 Mon Sep 17 00:00:00 2001 From: jingwei zhuang Date: Mon, 15 Aug 2022 12:53:06 -0700 Subject: [PATCH 343/376] updates --- cmd/root.go | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/cmd/root.go b/cmd/root.go index 3e349f4c..a5065b92 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -55,7 +55,7 @@ var ( dataResultFile string constructionResultFile string dataDirectory string - turnOnMemoryMode bool + inOnMemoryMode bool tableSize int64 // Config is the populated *configuration.Configuration from @@ -274,10 +274,10 @@ default values.`, ) checkDataCmd.Flags().BoolVar( - &turnOnMemoryMode, - "turn-on-memory-mode", + &inOnMemoryMode, + "in-memory-mode", false, - "turn-on-memory-mode configures badger DB inMeomry option. if turn-on-memory-mode=true This will override the all_in_memory_enabled", + "in-memory-mode configures badger DB inMeomry option. Only when in-memory-mode=true This will override the all_in_memory_enabled", ) rootCmd.AddCommand(checkDataCmd) @@ -390,8 +390,8 @@ func initConfig() { Config.DataDirectory = dataDirectory } - if turnOnMemoryMode { - Config.AllInMemoryEnabled = turnOnMemoryMode + if inOnMemoryMode { + Config.AllInMemoryEnabled = inOnMemoryMode } if tableSize != 0 { From 0af17a2104eccc63168761e8edb6cbcbbf557a36 Mon Sep 17 00:00:00 2001 From: jingwei zhuang Date: Mon, 15 Aug 2022 13:21:28 -0700 Subject: [PATCH 344/376] updates --- cmd/root.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/cmd/root.go b/cmd/root.go index a5065b92..6507a74d 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -55,7 +55,7 @@ var ( dataResultFile string constructionResultFile string dataDirectory string - inOnMemoryMode bool + inMemoryMode bool tableSize int64 // Config is the populated *configuration.Configuration from @@ -274,7 +274,7 @@ default values.`, ) checkDataCmd.Flags().BoolVar( - &inOnMemoryMode, + &inMemoryMode, "in-memory-mode", false, "in-memory-mode configures badger DB inMeomry option. Only when in-memory-mode=true This will override the all_in_memory_enabled", @@ -390,8 +390,8 @@ func initConfig() { Config.DataDirectory = dataDirectory } - if inOnMemoryMode { - Config.AllInMemoryEnabled = inOnMemoryMode + if inMemoryMode { + Config.AllInMemoryEnabled = inMemoryMode } if tableSize != 0 { From bd732bce2183e7f9bf4cd840be833416cf5164be Mon Sep 17 00:00:00 2001 From: jingwei zhuang Date: Mon, 15 Aug 2022 13:29:45 -0700 Subject: [PATCH 345/376] updates --- cmd/root.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cmd/root.go b/cmd/root.go index 6507a74d..a6b6cb6d 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -270,14 +270,14 @@ default values.`, &tableSize, "table-size", 0, - `Table-size configures the TableSize for badger DB. If table-size != 0, this will override the table_size from configuration file`, + "Table-size configures the TableSize for badger DB. If table-size != 0, this will override the table_size from configuration file", ) checkDataCmd.Flags().BoolVar( &inMemoryMode, "in-memory-mode", false, - "in-memory-mode configures badger DB inMeomry option. Only when in-memory-mode=true This will override the all_in_memory_enabled", + "in-memory-mode configures badger DB inMeomry option. Only when in-memory-mode=true, this will override the all_in_memory_enabled", ) rootCmd.AddCommand(checkDataCmd) From 5b745b9dda18bf33536df3af188ab3e84454b9bd Mon Sep 17 00:00:00 2001 From: jingweicb <96205264+jingweicb@users.noreply.github.com> Date: Tue, 16 Aug 2022 16:06:03 -0700 Subject: [PATCH 346/376] table size value validation (#343) table-size inputs validation --- cmd/root.go | 12 +++++++----- configuration/configuration.go | 8 ++++++++ 2 files changed, 15 insertions(+), 5 deletions(-) diff --git a/cmd/root.go b/cmd/root.go index a6b6cb6d..978c4b3d 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -242,7 +242,7 @@ default values.`, &startIndex, "start-block", -1, - `start-block is the block height to start syncing from. This will override the start_index from configuration file`, + `Start-block is the block height to start syncing from. This will override the start_index from configuration file`, ) checkDataCmd.Flags().Int64Var( @@ -269,15 +269,15 @@ default values.`, checkDataCmd.Flags().Int64Var( &tableSize, "table-size", - 0, - "Table-size configures the TableSize for badger DB. If table-size != 0, this will override the table_size from configuration file", + -1, + "Table-size configures the TableSize for badger DB. If table-size != -1, this will override the table_size from configuration file", ) checkDataCmd.Flags().BoolVar( &inMemoryMode, "in-memory-mode", false, - "in-memory-mode configures badger DB inMeomry option. Only when in-memory-mode=true, this will override the all_in_memory_enabled", + "In-memory-mode configures badger DB inMeomry option. Only when in-memory-mode=true, this will override the all_in_memory_enabled", ) rootCmd.AddCommand(checkDataCmd) @@ -394,8 +394,10 @@ func initConfig() { Config.AllInMemoryEnabled = inMemoryMode } - if tableSize != 0 { + if tableSize >= 2 && tableSize <= 100 { Config.TableSize = &tableSize + } else if tableSize != -1 { + log.Fatalf("table-size %d is not in the range [2, 100], please check your input", tableSize) } } diff --git a/configuration/configuration.go b/configuration/configuration.go index 70fb9c9d..02774a7d 100644 --- a/configuration/configuration.go +++ b/configuration/configuration.go @@ -353,6 +353,14 @@ func assertConfiguration(ctx context.Context, config *Configuration) error { return errors.New("serial_block_workers must be > 0") } + if config.TableSize != nil && (*config.TableSize < 2 || *config.TableSize > 100) { + return fmt.Errorf("table_size %d is not in the range [2, 100], please check your input", *config.TableSize) + } + + if config.ValueLogFileSize != nil && (*config.ValueLogFileSize < 128 || *config.ValueLogFileSize > 2048) { + return fmt.Errorf("value_log_file_size %d is not in the range [128, 2048], please check your input", *config.ValueLogFileSize) + } + if err := assertDataConfiguration(config.Data); err != nil { return fmt.Errorf("%w: invalid data configuration", err) } From 4ee4a2ab0d6eef43780fe071effabc015dade27a Mon Sep 17 00:00:00 2001 From: jingweicb <96205264+jingweicb@users.noreply.github.com> Date: Thu, 18 Aug 2022 09:28:04 -0700 Subject: [PATCH 347/376] change tablesize default value (#344) --- configuration/types.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configuration/types.go b/configuration/types.go index 2534cbc3..82afa7b3 100644 --- a/configuration/types.go +++ b/configuration/types.go @@ -421,7 +421,7 @@ type Configuration struct { // TableSize unit is GB, enable users to define MaxTableSize // when AllInMemoryEnabled == true or MemoryLimitDisabled== true, Cli will look up this config - // default value is 6, modification range is [2, 100] + // default value is 3, modification range is [2, 100] TableSize *int64 `json:"table_size,omitempty"` // TableSize unit is MB, enable users to define ValueLogFileSize From 5cb502ca467a599a70981b61b93e0b108c0f7ec1 Mon Sep 17 00:00:00 2001 From: Jingfu Wang Date: Sun, 21 Aug 2022 22:46:26 -0400 Subject: [PATCH 348/376] feat: update version Signed-off-by: Jingfu Wang --- Dockerfile | 2 +- cmd/root.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 2e184edc..928f3921 100644 --- a/Dockerfile +++ b/Dockerfile @@ -31,7 +31,7 @@ RUN mkdir -p "$GOPATH/src" "$GOPATH/bin" && chmod -R 777 "$GOPATH" WORKDIR /go/src -ARG VERSION=v0.8.0 +ARG VERSION=v0.9.0 RUN git clone https://github.com/coinbase/rosetta-cli.git && \ cd rosetta-cli && \ git fetch --all --tags && \ diff --git a/cmd/root.go b/cmd/root.go index 978c4b3d..48eba524 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -434,6 +434,6 @@ var versionCmd = &cobra.Command{ Use: "version", Short: "Print rosetta-cli version", Run: func(cmd *cobra.Command, args []string) { - fmt.Println("v0.8.0") + fmt.Println("v0.9.0") }, } From 143959f73c7c51e4efe2fa7de481ba8ea4e1f0c0 Mon Sep 17 00:00:00 2001 From: Jingfu Wang Date: Fri, 9 Sep 2022 15:55:16 -0400 Subject: [PATCH 349/376] feat: error handling for configuration and result Signed-off-by: Jingfu Wang --- configuration/configuration.go | 66 +++++++++++++---------------- go.mod | 4 +- go.sum | 32 +++++++------- pkg/errors/errors.go | 44 ++++++++++--------- pkg/processor/reconciler_handler.go | 6 +-- pkg/results/construction_results.go | 2 +- pkg/results/data_results.go | 9 ++-- pkg/results/data_results_test.go | 21 ++------- pkg/results/types.go | 11 ----- pkg/results/utils.go | 10 ++--- pkg/results/utils_test.go | 2 +- 11 files changed, 88 insertions(+), 119 deletions(-) diff --git a/configuration/configuration.go b/configuration/configuration.go index 02774a7d..b2128a93 100644 --- a/configuration/configuration.go +++ b/configuration/configuration.go @@ -17,14 +17,13 @@ package configuration import ( "context" "encoding/hex" - "errors" "fmt" "log" "path" "runtime" "strings" - customerrors "github.com/coinbase/rosetta-cli/pkg/errors" + cliErrs "github.com/coinbase/rosetta-cli/pkg/errors" "github.com/coinbase/rosetta-sdk-go/asserter" "github.com/coinbase/rosetta-sdk-go/constructor/dsl" "github.com/coinbase/rosetta-sdk-go/constructor/job" @@ -213,11 +212,11 @@ func assertConstructionConfiguration(ctx context.Context, config *ConstructionCo } if len(config.Workflows) > 0 && len(config.ConstructorDSLFile) > 0 { - return fmt.Errorf("%w: cannot populate both workflows and DSL file path", customerrors.ErrParseFileFailed) + return cliErrs.ErrMultipleDSLFiles } if len(config.Workflows) == 0 && len(config.ConstructorDSLFile) == 0 { - return fmt.Errorf("%w: both workflows and DSL file path are empty", customerrors.ErrParseFileFailed) + return cliErrs.ErrNoDSLFile } // Compile ConstructorDSLFile and save to Workflows @@ -225,7 +224,7 @@ func assertConstructionConfiguration(ctx context.Context, config *ConstructionCo compiledWorkflows, err := dsl.Parse(ctx, config.ConstructorDSLFile) if err != nil { err.Log() - return fmt.Errorf("%w: compilation failed", err.Err) + return fmt.Errorf("DSL file is invalid, line %d, line contents %s: %w", err.Line, err.LineContents, err.Err) } config.Workflows = compiledWorkflows @@ -236,10 +235,10 @@ func assertConstructionConfiguration(ctx context.Context, config *ConstructionCo if workflow.Name == string(job.CreateAccount) || workflow.Name == string(job.RequestFunds) { if workflow.Concurrency != job.ReservedWorkflowConcurrency { return fmt.Errorf( - "%w: reserved workflow %s must have concurrency %d", - customerrors.ErrParseWorkflowFailed, + "DSL file is invalid, reserved workflow %s must have concurrency %d: %w", workflow.Name, job.ReservedWorkflowConcurrency, + cliErrs.ErrWrongWorkflowConcurrency, ) } } @@ -250,20 +249,20 @@ func assertConstructionConfiguration(ctx context.Context, config *ConstructionCo _, err := hex.DecodeString(account.PrivateKeyHex) if err != nil { return fmt.Errorf( - "%w: private key %s is not hex encoded for prefunded account", - err, + "private key %s is not hex encoded for prefunded account: %w", account.PrivateKeyHex, + err, ) } // Checks if valid CurveType if err := asserter.CurveType(account.CurveType); err != nil { - return fmt.Errorf("%w: invalid CurveType for prefunded account", err) + return fmt.Errorf("prefunded account curve type %s is invalid: %w", types.PrintStruct(account.CurveType), err) } // Checks if valid AccountIdentifier if err := asserter.AccountIdentifier(account.AccountIdentifier); err != nil { - return fmt.Errorf("Account.Address is missing for prefunded account") + return fmt.Errorf("prefunded account identifier %s is invalid: %w", types.PrintStruct(account.AccountIdentifier), err) } // Check if valid Currency when Currency is specified @@ -272,7 +271,7 @@ func assertConstructionConfiguration(ctx context.Context, config *ConstructionCo if account.Currency != nil { err = asserter.Currency(account.Currency) if err != nil { - return fmt.Errorf("%w: invalid currency for prefunded account", err) + return fmt.Errorf("prefunded account currency %s is invalid: %w", types.PrintStruct(account.Currency), err) } } } @@ -282,11 +281,11 @@ func assertConstructionConfiguration(ctx context.Context, config *ConstructionCo func assertDataConfiguration(config *DataConfiguration) error { // nolint:gocognit if config.StartIndex != nil && *config.StartIndex < 0 { - return fmt.Errorf("start index %d cannot be negative", *config.StartIndex) + return fmt.Errorf("start index %d is invalid: %w", *config.StartIndex, cliErrs.ErrNegativeStartIndex) } if !config.ReconciliationDisabled && config.BalanceTrackingDisabled { - return errors.New("balance tracking must be enabled to perform reconciliation") + return cliErrs.ErrBalanceTrackingIsDisabledForReconciliation } if config.EndConditions == nil { @@ -295,45 +294,40 @@ func assertDataConfiguration(config *DataConfiguration) error { // nolint:gocogn if config.EndConditions.Index != nil { if *config.EndConditions.Index < 0 { - return fmt.Errorf("end index %d cannot be negative", *config.EndConditions.Index) + return fmt.Errorf("end index %d is invalid: %w", *config.EndConditions.Index, cliErrs.ErrNegativeEndIndex) } } if config.EndConditions.ReconciliationCoverage != nil { coverage := config.EndConditions.ReconciliationCoverage.Coverage if coverage < 0 || coverage > 1 { - return fmt.Errorf("reconciliation coverage %f must be [0.0,1.0]", coverage) + return fmt.Errorf("reconciliation coverage %f is invalid: %w", coverage, cliErrs.ErrReconciliationOutOfRange) } index := config.EndConditions.ReconciliationCoverage.Index if index != nil && *index < 0 { - return fmt.Errorf("reconciliation coverage height %d must be >= 0", *index) + return fmt.Errorf("reconciliation coverage index %d is invalid: %w", *index, cliErrs.ErrNegativeReconciliationCoverageIndex) } accountCount := config.EndConditions.ReconciliationCoverage.AccountCount if accountCount != nil && *accountCount < 0 { return fmt.Errorf( - "reconciliation coverage account count %d must be >= 0", + "reconciliation coverage account count %d is invalid: %w", *accountCount, + cliErrs.ErrNegativeReconciliationCoverageAccountCount, ) } if config.BalanceTrackingDisabled { - return errors.New( - "balance tracking must be enabled for reconciliation coverage end condition", - ) + return cliErrs.ErrBalanceTrackingIsDisabledForReconciliationCoverageEndCondition } if config.IgnoreReconciliationError { - return errors.New( - "reconciliation errors cannot be ignored for reconciliation coverage end condition", - ) + return cliErrs.ErrReconciliationErrorIsIgnoredForReconciliationCoverageEndCondition } if config.ReconciliationDisabled { - return errors.New( - "reconciliation cannot be disabled for reconciliation coverage end condition", - ) + return cliErrs.ErrReconciliationIsDisabledForReconciliationCoverageEndCondition } } @@ -342,31 +336,31 @@ func assertDataConfiguration(config *DataConfiguration) error { // nolint:gocogn func assertConfiguration(ctx context.Context, config *Configuration) error { if err := asserter.NetworkIdentifier(config.Network); err != nil { - return fmt.Errorf("%w: invalid network identifier", err) + return fmt.Errorf("invalid network identifier %s: %w", types.PrintStruct(config.Network), err) } if config.SeenBlockWorkers <= 0 { - return errors.New("seen_block_workers must be > 0") + return fmt.Errorf("the number of seen block workers %d is invalid: %w", config.SeenBlockWorkers, cliErrs.ErrNegativeSeenBlockWorkers) } if config.SerialBlockWorkers <= 0 { - return errors.New("serial_block_workers must be > 0") + return fmt.Errorf("the number of serial block workers %d is invalid: %w", config.SerialBlockWorkers, cliErrs.ErrNegativeSerialBlockWorkers) } if config.TableSize != nil && (*config.TableSize < 2 || *config.TableSize > 100) { - return fmt.Errorf("table_size %d is not in the range [2, 100], please check your input", *config.TableSize) + return fmt.Errorf("table size %d is invalid: %w", *config.TableSize, cliErrs.ErrTableSizeIsOutOfRange) } if config.ValueLogFileSize != nil && (*config.ValueLogFileSize < 128 || *config.ValueLogFileSize > 2048) { - return fmt.Errorf("value_log_file_size %d is not in the range [128, 2048], please check your input", *config.ValueLogFileSize) + return fmt.Errorf("value log file size %d is invalid: %w", *config.ValueLogFileSize, cliErrs.ErrValueLogFileSizeIsOutOfRange) } if err := assertDataConfiguration(config.Data); err != nil { - return fmt.Errorf("%w: invalid data configuration", err) + return fmt.Errorf("data configuration is invalid: %w", err) } if err := assertConstructionConfiguration(ctx, config.Construction); err != nil { - return fmt.Errorf("%w: invalid construction configuration", err) + return fmt.Errorf("construction configuration is invalid: %w", err) } return nil @@ -410,7 +404,7 @@ func modifyFilePaths(config *Configuration, fileDir string) { func LoadConfiguration(ctx context.Context, filePath string) (*Configuration, error) { var configRaw Configuration if err := utils.LoadAndParse(filePath, &configRaw); err != nil { - return nil, fmt.Errorf("%w: unable to open configuration file", err) + return nil, fmt.Errorf("unable to load and parse configuration file: %w", err) } config := populateMissingFields(&configRaw) @@ -421,7 +415,7 @@ func LoadConfiguration(ctx context.Context, filePath string) (*Configuration, er modifyFilePaths(config, fileDir) if err := assertConfiguration(ctx, config); err != nil { - return nil, fmt.Errorf("%w: invalid configuration", err) + return nil, fmt.Errorf("configuration is invalid: %w", err) } color.Cyan( diff --git a/go.mod b/go.mod index dcb81ec5..1b0daef1 100644 --- a/go.mod +++ b/go.mod @@ -3,9 +3,8 @@ module github.com/coinbase/rosetta-cli go 1.16 require ( - github.com/coinbase/rosetta-sdk-go v0.7.11 + github.com/coinbase/rosetta-sdk-go v0.8.0 github.com/fatih/color v1.13.0 - github.com/golang/protobuf v1.5.2 // indirect github.com/google/go-cmp v0.5.6 // indirect github.com/mattn/go-colorable v0.1.12 // indirect github.com/olekukonko/tablewriter v0.0.5 @@ -14,6 +13,5 @@ require ( github.com/stretchr/testify v1.7.2 go.uber.org/zap v1.21.0 golang.org/x/sync v0.0.0-20210220032951-036812b2e83c - golang.org/x/sys v0.0.0-20211205182925-97ca703d548d // indirect google.golang.org/protobuf v1.27.1 // indirect ) diff --git a/go.sum b/go.sum index 4db8ed05..b8dc4796 100644 --- a/go.sum +++ b/go.sum @@ -26,6 +26,7 @@ github.com/Azure/azure-sdk-for-go/sdk/azcore v0.21.1/go.mod h1:fBF9PQNqB8scdgpZ3 github.com/Azure/azure-sdk-for-go/sdk/internal v0.8.3/go.mod h1:KLF4gFr6DcKFZwSuH8w8yEK6DpFl3LP5rhdvAb7Yz5I= github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v0.3.0/go.mod h1:tPaiy8S5bQ+S5sOiDlINkp7+Ef339+Nz5L5XO+cnOHo= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/BurntSushi/toml v1.1.0/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/DATA-DOG/go-sqlmock v1.3.3/go.mod h1:f/Ixk793poVmq4qj/V1dPUg2JEAKC73Q5eFN3EC/SaM= github.com/DataDog/zstd v1.5.2 h1:vUG4lAyuPCXO0TLbXvPv7EB7cNK1QV/luu55UHLrrn8= @@ -97,8 +98,8 @@ github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDk github.com/cloudflare/cloudflare-go v0.14.0/go.mod h1:EnwdgGMaFOruiPZRFSgn+TsQ3hQ7C/YWzIGLeu5c304= github.com/coinbase/kryptology v1.8.0 h1:Aoq4gdTsJhSU3lNWsD5BWmFSz2pE0GlmrljaOxepdYY= github.com/coinbase/kryptology v1.8.0/go.mod h1:RYXOAPdzOGUe3qlSFkMGn58i3xUA8hmxYHksuq+8ciI= -github.com/coinbase/rosetta-sdk-go v0.7.11 h1:T7camDGstlX2ENVE0QHR1AVu3asSXED0vg1xDaF6bYE= -github.com/coinbase/rosetta-sdk-go v0.7.11/go.mod h1:HLTqSTSnOGLWHGTxoUJQO2TLuKkas1B9i/7ByerK6lM= +github.com/coinbase/rosetta-sdk-go v0.8.0 h1:FXootIYNrQyVbpGfWfmEOBzEGKku8Wuyql+iyIy3L+g= +github.com/coinbase/rosetta-sdk-go v0.8.0/go.mod h1:tXPR6AIW9ogsH4tYIaFOKOgfJNanCvcyl7JKLd4DToc= github.com/consensys/bavard v0.1.8-0.20210406032232-f3452dc9b572/go.mod h1:Bpd0/3mZuaj6Sj+PqrmIquiOKy397AKGThQPaGzNXAQ= github.com/consensys/bavard v0.1.8-0.20210915155054-088da2f7f54a/go.mod h1:9ItSMtA/dXMAiL7BG6bqW2m3NdSEObYWoH223nGHukI= github.com/consensys/gnark-crypto v0.4.1-0.20210426202927-39ac3d4b3f1f/go.mod h1:815PAHg3wvysy0SyIqanF8gZ0Y1wjk/hrDHD/iT88+Q= @@ -110,6 +111,7 @@ github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3Ee github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE= github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/cpuguy83/go-md2man/v2 v2.0.1/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= +github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/cyberdelia/templates v0.0.0-20141128023046-ca7fffd4298c/go.mod h1:GyV+0YP4qX0UQ7r2MoYZ+AvYDp12OF5yg4q8rGnyNh4= github.com/dave/jennifer v1.2.0/go.mod h1:fIb+770HOpJ2fmN9EPPKOqm1vMGhB+TwXKMZhrIygKg= @@ -146,8 +148,8 @@ github.com/eclipse/paho.mqtt.golang v1.2.0/go.mod h1:H9keYFcgq3Qr5OUJm/JZI/i6U7j github.com/edsrzf/mmap-go v1.0.0/go.mod h1:YO35OhQPt3KJa3ryjFM5Bs14WD66h8eGKpfaBNrHW5M= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= -github.com/ethereum/go-ethereum v1.10.19 h1:EOR5JbL4MD5yeOqv8W2iC1s4NximrTjqFccUz8lyBRA= -github.com/ethereum/go-ethereum v1.10.19/go.mod h1:IJBNMtzKcNHPtllYihy6BL2IgK1u+32JriaTbdt4v+w= +github.com/ethereum/go-ethereum v1.10.21 h1:5lqsEx92ZaZzRyOqBEXux4/UR06m296RGzN3ol3teJY= +github.com/ethereum/go-ethereum v1.10.21/go.mod h1:EYFyF19u3ezGLD4RqOkLq+ZCXzYbLoNDdZlMt7kyKFg= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/fatih/color v1.13.0 h1:8LOYc1KYPPmyKMuN8QV2DNRWNbLo6LZ0iLs8+mlH53w= github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk= @@ -195,9 +197,7 @@ github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:x github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= -github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= -github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw= github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= @@ -214,7 +214,6 @@ github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMyw github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.6 h1:BKbKCqvP6I+rmFHt06ZmyQtvB8xAkWdhFyr0ZUNZcxQ= @@ -443,6 +442,7 @@ github.com/tyler-smith/go-bip39 v1.0.1-0.20181017060643-dbb3b84ba2ef/go.mod h1:s github.com/tyler-smith/go-bip39 v1.0.2/go.mod h1:sJ5fKU0s6JVwZjjcUEX2zFOnvq0ASQ2K9Zr6cf67kNs= github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= github.com/urfave/cli/v2 v2.3.0/go.mod h1:LJmUH05zAU44vOAcrfzZQKsZbVcdbOG8rtL3/XcUArI= +github.com/urfave/cli/v2 v2.10.2/go.mod h1:f8iq5LtQ/bLxafbdBSLPPNsgaW0l/2fYYEHhAyPlwvo= github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= github.com/valyala/fasttemplate v1.0.1/go.mod h1:UQGH1tvbgY+Nz5t2n7tXsz52dQxojPUpymEIMZ47gx8= github.com/valyala/fasttemplate v1.2.1/go.mod h1:KHLXt3tVN2HBp8eijSv/kGJopbvo7S+qRAEEKiv+SiQ= @@ -453,6 +453,7 @@ github.com/vmihailenco/tagparser/v2 v2.0.0/go.mod h1:Wri+At7QHww0WTrCBeu4J6bNtoV github.com/willf/bitset v1.1.3/go.mod h1:RjeCKbqT1RxIR/KWY6phxZiaY1IyutSBfGjNPySAYV4= github.com/xlab/treeprint v0.0.0-20180616005107-d6fb6747feb6/go.mod h1:ce1O1j6UtZfjr22oyGxGLbauSBp2YVXpARAosm7dHBg= github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= +github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673/go.mod h1:N3UwUGtsrSj3ccvlPHLoLsHnpR27oXr4ZE984MbSER8= github.com/ybbus/jsonrpc v2.1.2+incompatible/go.mod h1:XJrh1eMSzdIYFbM08flv0wp5G35eRniyeGut1z+LSiE= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= @@ -547,8 +548,9 @@ golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= golang.org/x/net v0.0.0-20210610132358-84b48f89b13b/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f h1:OfiFi4JbukWwe3lzw+xunroH1mnC1e2Gy5cxNJApiSY= golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20220607020251-c690dde0001d h1:4SFsTMi4UahlKoloni7L4eYzhFRifURQLw+yv0QDCx8= +golang.org/x/net v0.0.0-20220607020251-c690dde0001d/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -608,10 +610,11 @@ golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211019181941-9d821ace8654/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20211205182925-97ca703d548d h1:FjkYO/PPp4Wi0EAUOVLxePm7qVW4r4ctbWpURyuOD0E= -golang.org/x/sys v0.0.0-20211205182925-97ca703d548d/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a h1:dGzPydgVsqGcTRVwiLJ1jVbufYwmzD3LfVPLKsKg+0k= +golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= @@ -657,8 +660,9 @@ golang.org/x/tools v0.1.8-0.20211029000441-d6a9af8af023/go.mod h1:nABZi5QlRsZVlz golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20220517211312-f3a8303e98df h1:5Pf6pFKu98ODmgnpvkJ3kFUOQGGLIzLIkbzUHp47618= +golang.org/x/xerrors v0.0.0-20220517211312-f3a8303e98df/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8= gonum.org/v1/gonum v0.0.0-20180816165407-929014505bf4/go.mod h1:Y+Yx5eoAFn32cQvJDxZx5Dpnq+c3wtXuadVZAcxbbBo= gonum.org/v1/gonum v0.0.0-20181121035319-3f7ecaa7e8ca/go.mod h1:Y+Yx5eoAFn32cQvJDxZx5Dpnq+c3wtXuadVZAcxbbBo= gonum.org/v1/gonum v0.6.0/go.mod h1:9mxDZsDKxgMAuccQkewq682L+0eCu4dCN2yonUJTCLU= @@ -691,22 +695,17 @@ google.golang.org/genproto v0.0.0-20191115194625-c23dd37a84c9/go.mod h1:n3cpQtvx google.golang.org/genproto v0.0.0-20191216164720-4f79533eabd1/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= google.golang.org/genproto v0.0.0-20191230161307-f3c370f40bfb/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= google.golang.org/genproto v0.0.0-20200108215221-bd8f9a0ef82f/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= -google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= -google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= -google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= -google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.27.1 h1:SnqbnDw1V7RiZcXPx5MEeqPv2s79L9i7BJUlG/+RurQ= @@ -721,7 +720,6 @@ gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/natefinch/npipe.v2 v2.0.0-20160621034901-c1b8fa8bdcce/go.mod h1:5AcXVHNjg+BDxry382+8OKon8SEWiKktQR07RKPsv1c= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= -gopkg.in/urfave/cli.v1 v1.20.0/go.mod h1:vuBzUtMdQeixQj8LVd+/98pzhxNGQoyuPBlsXHOQNO0= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= diff --git a/pkg/errors/errors.go b/pkg/errors/errors.go index a5f93b93..1b608be2 100644 --- a/pkg/errors/errors.go +++ b/pkg/errors/errors.go @@ -18,32 +18,36 @@ import ( "errors" ) -// Configuration Errors - var ( - // Data Errors - - ErrCheckStorageTipFailed = errors.New("unable to check storage tip") + // Configuration errors + ErrMultipleDSLFiles = errors.New("multiple DSL files are found") + ErrNoDSLFile = errors.New("no DSL file") + ErrWrongWorkflowConcurrency = errors.New("reserved workflow concurrency doesn't match") + ErrNegativeStartIndex = errors.New("start index is negative") + ErrNegativeEndIndex = errors.New("end index is negative") + ErrNegativeReconciliationCoverageIndex = errors.New("reconciliation coverage index is negative") + ErrNegativeReconciliationCoverageAccountCount = errors.New("reconciliation coverage account is negative") + ErrNegativeSeenBlockWorkers = errors.New("the number of seen block workers is negative") + ErrNegativeSerialBlockWorkers = errors.New("the number of serial block workers is negative") + ErrReconciliationOutOfRange = errors.New("reconciliation is out of range, it must be in the range [0, 1]") + ErrTableSizeIsOutOfRange = errors.New("table size is out of range, it must be in the range [2, 100]") + ErrValueLogFileSizeIsOutOfRange = errors.New("value log file size is out of range, it must be in the range [128, 2048]") + ErrBalanceTrackingIsDisabledForReconciliation = errors.New("balance tracking cannot be disabled for reconciliation") + ErrBalanceTrackingIsDisabledForReconciliationCoverageEndCondition = errors.New("balance tracking cannot be disabled for reconciliation coverage end condition") + ErrReconciliationErrorIsIgnoredForReconciliationCoverageEndCondition = errors.New("reconciliation error cannot be ignored for reconciliation coverage end condition") + ErrReconciliationIsDisabledForReconciliationCoverageEndCondition = errors.New("reconciliation cannot be disabled for reconciliation coverage end condition") + ErrConstructionConfigMissing = errors.New("construction configuration is missing") + + // Data check errors ErrDataCheckHalt = errors.New("data check halted") ErrInitDataTester = errors.New("unexpected error occurred while trying to initialize data tester") - - // Construction Configuration Errors - - ErrParseFileFailed = errors.New("unable to parse config files") - ErrBalanceTrackingDisabled = errors.New("balance tracking disabled") - ErrReconciliationConfig = errors.New("invalid reconciliation error") - ErrCompileDSLFileFailed = errors.New("unable to compile DSL file") - ErrParseWorkflowFailed = errors.New("unable to parse workflow") - ErrConstructionConfigMissing = errors.New("construction configuration is missing") + ErrReconciliationFailure = errors.New("reconciliation failure") // Construction check errors - ErrConstructionCheckHalt = errors.New("construction check halted") + // Command errors + ErrBlockNotFound = errors.New("block not found") + ErrNoAvailableNetwork = errors.New("no networks available") ErrAsserterConfigError = errors.New("asserter configuration validation failed") - - // Bad Command Errors - - ErrBlockNotFound = errors.New("block not found") - ErrNoAvailableNetwork = errors.New("no networks available") ) diff --git a/pkg/processor/reconciler_handler.go b/pkg/processor/reconciler_handler.go index 13521500..f7148b74 100644 --- a/pkg/processor/reconciler_handler.go +++ b/pkg/processor/reconciler_handler.go @@ -22,8 +22,8 @@ import ( "time" "github.com/coinbase/rosetta-cli/pkg/logger" - "github.com/coinbase/rosetta-cli/pkg/results" + cliErrs "github.com/coinbase/rosetta-cli/pkg/errors" "github.com/coinbase/rosetta-sdk-go/reconciler" "github.com/coinbase/rosetta-sdk-go/storage/modules" "github.com/coinbase/rosetta-sdk-go/types" @@ -162,7 +162,7 @@ func (h *ReconcilerHandler) ReconciliationFailed( h.InactiveFailureBlock = block return fmt.Errorf( "%w: inactive reconciliation error for %s at %d (computed: %s%s, live: %s%s)", - results.ErrReconciliationFailure, + cliErrs.ErrReconciliationFailure, account.Address, block.Index, computedBalance, @@ -176,7 +176,7 @@ func (h *ReconcilerHandler) ReconciliationFailed( h.ActiveFailureBlock = block return fmt.Errorf( "%w: active reconciliation error for %s at %d (computed: %s%s, live: %s%s)", - results.ErrReconciliationFailure, + cliErrs.ErrReconciliationFailure, account.Address, block.Index, computedBalance, diff --git a/pkg/results/construction_results.go b/pkg/results/construction_results.go index 4761a538..e578b263 100644 --- a/pkg/results/construction_results.go +++ b/pkg/results/construction_results.go @@ -287,7 +287,7 @@ func ComputeCheckConstructionStatus( func FetchCheckConstructionStatus(url string) (*CheckConstructionStatus, error) { var status CheckConstructionStatus if err := JSONFetch(url, &status); err != nil { - return nil, fmt.Errorf("%w: unable to fetch construction status", err) + return nil, fmt.Errorf("unable to fetch check construction status: %w", err) } return &status, nil diff --git a/pkg/results/data_results.go b/pkg/results/data_results.go index 1c2f98e6..6ba9251d 100644 --- a/pkg/results/data_results.go +++ b/pkg/results/data_results.go @@ -27,6 +27,7 @@ import ( "github.com/coinbase/rosetta-cli/configuration" + cliErrs "github.com/coinbase/rosetta-cli/pkg/errors" "github.com/coinbase/rosetta-sdk-go/asserter" "github.com/coinbase/rosetta-sdk-go/fetcher" "github.com/coinbase/rosetta-sdk-go/reconciler" @@ -429,7 +430,7 @@ func ComputeCheckDataStatus( func FetchCheckDataStatus(url string) (*CheckDataStatus, error) { var status CheckDataStatus if err := JSONFetch(url, &status); err != nil { - return nil, fmt.Errorf("%w: unable to fetch construction status", err) + return nil, fmt.Errorf("unable to fetch check data status: %w", err) } return &status, nil @@ -511,9 +512,7 @@ func (c *CheckDataTests) Print() { // a non-500 response. func RequestResponseTest(err error) bool { return !(fetcher.Err(err) || - errors.Is(err, utils.ErrNetworkNotSupported) || - errors.Is(err, syncer.ErrGetNetworkStatusFailed) || - errors.Is(err, syncer.ErrFetchBlockFailed)) + errors.Is(err, utils.ErrNetworkNotSupported)) } // ResponseAssertionTest returns a boolean @@ -569,7 +568,7 @@ func ReconciliationTest( reconciliationsPerformed bool, reconciliationsFailed bool, ) *bool { - if errors.Is(err, ErrReconciliationFailure) { + if errors.Is(err, cliErrs.ErrReconciliationFailure) { return &f } diff --git a/pkg/results/data_results_test.go b/pkg/results/data_results_test.go index 9d687817..b5816fc8 100644 --- a/pkg/results/data_results_test.go +++ b/pkg/results/data_results_test.go @@ -24,6 +24,7 @@ import ( "github.com/coinbase/rosetta-cli/configuration" + cliErrs "github.com/coinbase/rosetta-cli/pkg/errors" "github.com/coinbase/rosetta-sdk-go/asserter" "github.com/coinbase/rosetta-sdk-go/fetcher" sdkMocks "github.com/coinbase/rosetta-sdk-go/mocks/storage/modules" @@ -123,20 +124,6 @@ func TestComputeCheckDataResults(t *testing.T) { }, }, }, - "default configuration, no storage, syncer and fetch errors": { - cfg: configuration.DefaultConfiguration(), - err: []error{ - syncer.ErrGetNetworkStatusFailed, - syncer.ErrFetchBlockFailed, - }, - result: &CheckDataResults{ - Tests: &CheckDataTests{ - RequestResponse: false, - ResponseAssertion: true, - BlockSyncing: &f, - }, - }, - }, "default configuration, no storage, assertion errors": { cfg: configuration.DefaultConfiguration(), err: []error{asserter.ErrAmountValueMissing}, @@ -327,7 +314,7 @@ func TestComputeCheckDataResults(t *testing.T) { }, "default configuration, no storage, reconciliation errors": { cfg: configuration.DefaultConfiguration(), - err: []error{ErrReconciliationFailure}, + err: []error{cliErrs.ErrReconciliationFailure}, result: &CheckDataResults{ Tests: &CheckDataTests{ RequestResponse: true, @@ -338,7 +325,7 @@ func TestComputeCheckDataResults(t *testing.T) { }, "default configuration, counter storage, reconciliation errors": { cfg: configuration.DefaultConfiguration(), - err: []error{ErrReconciliationFailure}, + err: []error{cliErrs.ErrReconciliationFailure}, provideCounterStorage: true, activeReconciliations: 10, reconciliationFailures: 19, @@ -387,7 +374,7 @@ func TestComputeCheckDataResults(t *testing.T) { var testErr error if err != nil { testName = err.Error() - testErr = fmt.Errorf("%w: test wrapping", err) + testErr = fmt.Errorf("test wrapping: %w", err) test.result.Error = testErr.Error() } diff --git a/pkg/results/types.go b/pkg/results/types.go index 6057ef67..a4215928 100644 --- a/pkg/results/types.go +++ b/pkg/results/types.go @@ -14,18 +14,7 @@ package results -import ( - "errors" -) - const ( // TimeElapsedCounter tracks the total time elapsed in seconds. TimeElapsedCounter = "time_elapsed" ) - -var ( - // ErrReconciliationFailure is returned if reconciliation fails. - // TODO: Move to reconciler package (had to remove from processor - // to prevent circular dependency) - ErrReconciliationFailure = errors.New("reconciliation failure") -) diff --git a/pkg/results/utils.go b/pkg/results/utils.go index d9eeef41..e9de29af 100644 --- a/pkg/results/utils.go +++ b/pkg/results/utils.go @@ -21,26 +21,26 @@ import ( "net/http" ) -// JSONFetch makes a GET request to the URL and marshals +// JSONFetch makes a GET request to the URL and unmarshal // the response into output. func JSONFetch(url string, output interface{}) error { resp, err := http.Get(url) // #nosec if err != nil { - return fmt.Errorf("%w: unable to fetch GET %s", err, url) + return fmt.Errorf("unable to fetch url %s: %w", url, err) } defer resp.Body.Close() body, err := ioutil.ReadAll(resp.Body) if err != nil { - return fmt.Errorf("%w: unable to read body", err) + return fmt.Errorf("unable to read body: %w", err) } if resp.StatusCode != http.StatusOK { - return fmt.Errorf("received %d status with body %s", resp.StatusCode, body) + return fmt.Errorf("received %d status with body %s", resp.StatusCode, string(body)) } if err := json.Unmarshal(body, output); err != nil { - return fmt.Errorf("%w: unable to unmarshal JSON", err) + return fmt.Errorf("unable to unmarshal: %w", err) } return nil diff --git a/pkg/results/utils_test.go b/pkg/results/utils_test.go index ed63a9cd..8079600d 100644 --- a/pkg/results/utils_test.go +++ b/pkg/results/utils_test.go @@ -46,7 +46,7 @@ func TestJSONFetch(t *testing.T) { "not JSON": { status: http.StatusOK, body: `hello`, - expectedError: "invalid character 'h' looking for beginning of value: unable to unmarshal JSON", + expectedError: "unable to unmarshal: invalid character 'h' looking for beginning of value", }, } From 1913ad27a0602c5124f9cf9f31dcfa637944518a Mon Sep 17 00:00:00 2001 From: Jingfu Wang Date: Mon, 12 Sep 2022 12:49:28 -0400 Subject: [PATCH 350/376] feat: error handling for logger and cmd Signed-off-by: Jingfu Wang --- cmd/check_construction.go | 17 ++++--- cmd/check_data.go | 18 ++++--- cmd/check_spec.go | 40 ++++++++------- cmd/check_spec_utils.go | 14 ++---- cmd/configuration_create.go | 2 +- cmd/configuration_validate.go | 2 +- cmd/root.go | 10 ++-- cmd/utils_asserter_configuration.go | 6 +-- cmd/utils_train_zstd.go | 4 +- cmd/validate_asserter_config.go | 78 ++++++++++++----------------- cmd/view_balance.go | 12 ++--- cmd/view_block.go | 21 ++++---- cmd/view_networks.go | 9 ++-- pkg/errors/errors.go | 22 ++++++-- pkg/logger/logger.go | 49 +++++++++--------- 15 files changed, 154 insertions(+), 150 deletions(-) diff --git a/cmd/check_construction.go b/cmd/check_construction.go index 617a4a0d..497497d7 100644 --- a/cmd/check_construction.go +++ b/cmd/check_construction.go @@ -17,13 +17,15 @@ package cmd import ( "context" "fmt" - "github.com/coinbase/rosetta-cli/pkg/errors" "time" + cliErrs "github.com/coinbase/rosetta-cli/pkg/errors" + "github.com/coinbase/rosetta-cli/pkg/results" "github.com/coinbase/rosetta-cli/pkg/tester" "github.com/coinbase/rosetta-sdk-go/fetcher" + "github.com/coinbase/rosetta-sdk-go/types" "github.com/coinbase/rosetta-sdk-go/utils" "github.com/spf13/cobra" "golang.org/x/sync/errgroup" @@ -59,7 +61,7 @@ func runCheckConstructionCmd(_ *cobra.Command, _ []string) error { Config, nil, nil, - errors.ErrConstructionConfigMissing, + cliErrs.ErrConstructionConfigMissing, ) } @@ -88,7 +90,7 @@ func runCheckConstructionCmd(_ *cobra.Command, _ []string) error { Config, nil, nil, - fmt.Errorf("%w: unable to initialize asserter", fetchErr.Err), + fmt.Errorf("unable to initialize asserter for fetcher: %w", fetchErr.Err), ) } @@ -99,7 +101,7 @@ func runCheckConstructionCmd(_ *cobra.Command, _ []string) error { Config, nil, nil, - fmt.Errorf("%w: unable to confirm network is supported", err), + fmt.Errorf("unable to confirm network %s is supported: %w", types.PrintStruct(Config.Network), err), ) } @@ -112,7 +114,7 @@ func runCheckConstructionCmd(_ *cobra.Command, _ []string) error { Config, nil, nil, - err, + fmt.Errorf("network options don't match asserter configuration file %s: %w", asserterConfigurationFile, err), ) } } @@ -130,10 +132,9 @@ func runCheckConstructionCmd(_ *cobra.Command, _ []string) error { Config, nil, nil, - fmt.Errorf("%w: unable to initialize construction tester", err), + fmt.Errorf("unable to initialize construction tester: %w", err), ) } - defer constructionTester.CloseDatabase(ctx) if err := constructionTester.PerformBroadcasts(ctx); err != nil { @@ -141,7 +142,7 @@ func runCheckConstructionCmd(_ *cobra.Command, _ []string) error { Config, nil, nil, - fmt.Errorf("%w: unable to perform broadcasts", err), + fmt.Errorf("unable to perform broadcasts: %w", err), ) } diff --git a/cmd/check_data.go b/cmd/check_data.go index cc54433b..962f3647 100644 --- a/cmd/check_data.go +++ b/cmd/check_data.go @@ -17,12 +17,12 @@ package cmd import ( "context" "fmt" - "github.com/coinbase/rosetta-cli/pkg/errors" "time" "github.com/coinbase/rosetta-cli/pkg/results" "github.com/coinbase/rosetta-cli/pkg/tester" "github.com/coinbase/rosetta-sdk-go/fetcher" + "github.com/coinbase/rosetta-sdk-go/types" "github.com/coinbase/rosetta-sdk-go/utils" "github.com/spf13/cobra" "golang.org/x/sync/errgroup" @@ -96,7 +96,7 @@ func runCheckDataCmd(_ *cobra.Command, _ []string) error { Config, nil, nil, - fmt.Errorf("%w: unable to initialize asserter", fetchErr.Err), + fmt.Errorf("unable to initialize asserter for fetcher: %w", fetchErr.Err), "", "", ) @@ -109,7 +109,7 @@ func runCheckDataCmd(_ *cobra.Command, _ []string) error { Config, nil, nil, - fmt.Errorf("%w: unable to confirm network", err), + fmt.Errorf("unable to confirm network %s is supported: %w", types.PrintStruct(Config.Network), err), "", "", ) @@ -124,7 +124,7 @@ func runCheckDataCmd(_ *cobra.Command, _ []string) error { Config, nil, nil, - err, + fmt.Errorf("network options don't match asserter configuration file %s: %w", asserterConfigurationFile, err), "", "", ) @@ -141,9 +141,15 @@ func runCheckDataCmd(_ *cobra.Command, _ []string) error { nil, // only populated when doing recursive search &SignalReceived, ) - if err != nil { - return fmt.Errorf("%s:%s", errors.ErrInitDataTester, err) + return results.ExitData( + Config, + nil, + nil, + fmt.Errorf("unable to initialize data tester: %w", err), + "", + "", + ) } defer dataTester.CloseDatabase(ctx) diff --git a/cmd/check_spec.go b/cmd/check_spec.go index 206f1cf1..129b8c44 100644 --- a/cmd/check_spec.go +++ b/cmd/check_spec.go @@ -23,6 +23,8 @@ import ( "github.com/coinbase/rosetta-sdk-go/fetcher" "github.com/coinbase/rosetta-sdk-go/types" "github.com/spf13/cobra" + + cliErrs "github.com/coinbase/rosetta-cli/pkg/errors" ) var ( @@ -32,7 +34,7 @@ var ( Long: `Check:spec checks whether a Rosetta implementation satisfies either Coinbase-specific requirements or minimum requirements specified in rosetta-api.org. -By default, check:spec will verify only Coinbase spec requirements. To verifiy the minimum requirements as well, +By default, check:spec will verify only Coinbase spec requirements. To verify the minimum requirements as well, add the --all flag to the check:spec command: rosetta-cli check:spec --all --configuration-file [filepath] @@ -60,7 +62,7 @@ type checkSpec struct { func newCheckSpec(ctx context.Context) (*checkSpec, error) { if Config.Construction == nil { - return nil, fmt.Errorf("%v", errRosettaConfigNoConstruction) + return nil, cliErrs.ErrConstructionConfigMissing } onlineFetcherOpts := []fetcher.Option{ @@ -97,7 +99,7 @@ func newCheckSpec(ctx context.Context) (*checkSpec, error) { Config, nil, nil, - fmt.Errorf("%v: unable to initialize asserter for online node fetcher", fetchErr.Err), + fmt.Errorf("unable to initialize asserter for online fetcher: %w", fetchErr.Err), "", "", ) @@ -132,7 +134,7 @@ func (cs *checkSpec) networkOptions(ctx context.Context) checkSpecOutput { // This is an endpoint for offline mode _, err := cs.offlineFetcher.NetworkOptionsRetry(ctx, Config.Network, nil) if err != nil { - printError("%v: unable to fetch network options\n", err.Err) + printError("unable to fetch network options: %v\n", err.Err) markAllValidationsFailed(output) return output } @@ -169,7 +171,7 @@ func (cs *checkSpec) networkList(ctx context.Context) checkSpecOutput { // endpoint for offline mode if err != nil { - printError("%v: unable to fetch network list", err.Err) + printError("unable to fetch network list: %v\n", err.Err) markAllValidationsFailed(output) return output } @@ -211,12 +213,12 @@ func (cs *checkSpec) accountCoins(ctx context.Context) checkSpecOutput { if isUTXO() { acct, _, currencies, err := cs.getAccount(ctx) if err != nil { - printError("%v: unable to get an account\n", err) + printError("unable to get an account: %v\n", err) markAllValidationsFailed(output) return output } - if err != nil { - printError("%v\n", errAccountNullPointer) + if acct == nil { + printError("%v\n", cliErrs.ErrAccountNullPointer) markAllValidationsFailed(output) return output } @@ -228,7 +230,7 @@ func (cs *checkSpec) accountCoins(ctx context.Context) checkSpecOutput { false, currencies) if fetchErr != nil { - printError("%v: unable to get coins for account: %v\n", fetchErr.Err, *acct) + printError("unable to get coins for account %s: %v\n", types.PrintStruct(acct), fetchErr.Err) markAllValidationsFailed(output) return output } @@ -261,7 +263,7 @@ func (cs *checkSpec) block(ctx context.Context) checkSpecOutput { res, fetchErr := cs.onlineFetcher.NetworkStatusRetry(ctx, Config.Network, nil) if fetchErr != nil { - printError("%v: unable to get network status\n", fetchErr.Err) + printError("unable to get network status: %v\n", fetchErr.Err) markAllValidationsFailed(output) return output } @@ -278,7 +280,7 @@ func (cs *checkSpec) block(ctx context.Context) checkSpecOutput { } b, fetchErr := cs.onlineFetcher.BlockRetry(ctx, Config.Network, &blockID) if fetchErr != nil { - printError("%v: unable to fetch block %v\n", fetchErr.Err, blockID) + printError("unable to fetch block %s: %v\n", types.PrintStruct(blockID), fetchErr.Err) markAllValidationsFailed(output) return output } @@ -286,7 +288,7 @@ func (cs *checkSpec) block(ctx context.Context) checkSpecOutput { if block == nil { block = b } else if !isEqual(types.Hash(*block), types.Hash(*b)) { - printError("%v\n", errBlockNotIdempotent) + printError("%v\n", cliErrs.ErrBlockNotIdempotent) setValidationStatusFailed(output, idempotent) } } @@ -295,24 +297,24 @@ func (cs *checkSpec) block(ctx context.Context) checkSpecOutput { // fetch the tip block again res, fetchErr = cs.onlineFetcher.NetworkStatusRetry(ctx, Config.Network, nil) if fetchErr != nil { - printError("%v: unable to get network status\n", fetchErr.Err) + printError("unable to get network status: %v\n", fetchErr.Err) setValidationStatusFailed(output, defaultTip) return output } tip := res.CurrentBlockIdentifier - // tip shoud be returned if block_identifier is not specified + // tip should be returned if block_identifier is not specified emptyBlockID := &types.PartialBlockIdentifier{} block, fetchErr := cs.onlineFetcher.BlockRetry(ctx, Config.Network, emptyBlockID) if fetchErr != nil { - printError("%v: unable to fetch tip block\n", fetchErr.Err) + printError("unable to fetch tip block: %v\n", fetchErr.Err) setValidationStatusFailed(output, defaultTip) return output } // block index returned from /block should be >= the index returned by /network/status if isNegative(block.BlockIdentifier.Index - tip.Index) { - printError("%v\n", errBlockTip) + printError("%v\n", cliErrs.ErrBlockTip) setValidationStatusFailed(output, defaultTip) } @@ -383,7 +385,7 @@ func (cs *checkSpec) getAccount(ctx context.Context) ( error) { res, err := cs.onlineFetcher.NetworkStatusRetry(ctx, Config.Network, nil) if err != nil { - return nil, nil, nil, fmt.Errorf("%v: unable to get network status", err.Err) + return nil, nil, nil, fmt.Errorf("unable to get network status of network %s: %w", types.PrintStruct(Config.Network), err.Err) } var acct *types.AccountIdentifier @@ -399,7 +401,7 @@ func (cs *checkSpec) getAccount(ctx context.Context) ( block, err := cs.onlineFetcher.BlockRetry(ctx, Config.Network, blockID) if err != nil { - return nil, nil, nil, fmt.Errorf("%v: unable to fetch block at index: %v", err.Err, i) + return nil, nil, nil, fmt.Errorf("unable to fetch block at index %d: %w", i, err.Err) } // looking for an account in block transactions @@ -425,7 +427,7 @@ func runCheckSpecCmd(_ *cobra.Command, _ []string) error { ctx := context.Background() cs, err := newCheckSpec(ctx) if err != nil { - return fmt.Errorf("%v: unable to create checkSpec object with online URL", err) + return fmt.Errorf("unable to create checkSpec object with online URL: %w", err) } output := []checkSpecOutput{} diff --git a/cmd/check_spec_utils.go b/cmd/check_spec_utils.go index ab70d3f2..bc119727 100644 --- a/cmd/check_spec_utils.go +++ b/cmd/check_spec_utils.go @@ -15,21 +15,13 @@ package cmd import ( - "errors" "fmt" "strconv" "github.com/coinbase/rosetta-sdk-go/fetcher" "github.com/fatih/color" -) -var ( - errErrorEmptyMessage = errors.New("Error object can't have empty message") - errErrorNegativeCode = errors.New("Error object can't have negative code") - errAccountNullPointer = errors.New("Null pointer to Account object") - errBlockNotIdempotent = errors.New("Multiple calls with the same hash don't return the same block") - errBlockTip = errors.New("Unspecified block_identifier doesn't give the tip block") - errRosettaConfigNoConstruction = errors.New("No construction element in Rosetta config") + cliErrs "github.com/coinbase/rosetta-cli/pkg/errors" ) type checkSpecAPI string @@ -112,12 +104,12 @@ func setValidationStatusFailed(output checkSpecOutput, req checkSpecRequirement) func validateErrorObject(err *fetcher.Error, output checkSpecOutput) { if err != nil { if err.ClientErr != nil && isNegative(int64(err.ClientErr.Code)) { - printError("%v\n", errErrorNegativeCode) + printError("%v\n", cliErrs.ErrErrorNegativeCode) setValidationStatusFailed(output, errorCode) } if err.ClientErr != nil && isEmpty(err.ClientErr.Message) { - printError("%v\n", errErrorEmptyMessage) + printError("%v\n", cliErrs.ErrErrorEmptyMessage) setValidationStatusFailed(output, errorMessage) } } diff --git a/cmd/configuration_create.go b/cmd/configuration_create.go index 0bba6bc5..42c728ef 100644 --- a/cmd/configuration_create.go +++ b/cmd/configuration_create.go @@ -34,7 +34,7 @@ var ( func runConfigurationCreateCmd(cmd *cobra.Command, args []string) error { if err := utils.SerializeAndWrite(args[0], configuration.DefaultConfiguration()); err != nil { - return fmt.Errorf("%w: unable to save configuration file to %s", err, args[0]) + return fmt.Errorf("unable to save configuration file to %s: %w", args[0], err) } return nil diff --git a/cmd/configuration_validate.go b/cmd/configuration_validate.go index 80844519..6fb2c9d2 100644 --- a/cmd/configuration_validate.go +++ b/cmd/configuration_validate.go @@ -35,7 +35,7 @@ var ( func runConfigurationValidateCmd(cmd *cobra.Command, args []string) error { _, err := configuration.LoadConfiguration(Context, args[0]) if err != nil { - return fmt.Errorf("%w: configuration validation failed %s", err, args[0]) + return fmt.Errorf("configuration validation failed %s: %w", args[0], err) } color.Green("Configuration file validated!") diff --git a/cmd/root.go b/cmd/root.go index 48eba524..590eeddd 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -103,12 +103,12 @@ var ( // rootPreRun is executed before the root command runs and sets up cpu // profiling. // -// Bassed on https://golang.org/pkg/runtime/pprof/#hdr-Profiling_a_Go_program +// Based on https://golang.org/pkg/runtime/pprof/#hdr-Profiling_a_Go_program func rootPreRun(*cobra.Command, []string) error { if cpuProfile != "" { f, err := os.Create(path.Clean(cpuProfile)) if err != nil { - return fmt.Errorf("%w: unable to create CPU profile file", err) + return fmt.Errorf("unable to create CPU profile file: %w", err) } if err := pprof.StartCPUProfile(f); err != nil { if err := f.Close(); err != nil { @@ -129,7 +129,7 @@ func rootPreRun(*cobra.Command, []string) error { runtime.SetBlockProfileRate(1) f, err := os.Create(path.Clean(blockProfile)) if err != nil { - return fmt.Errorf("%w: unable to create block profile file", err) + return fmt.Errorf("unable to create block profile file: %w", err) } p := pprof.Lookup("block") @@ -355,7 +355,7 @@ func initConfig() { } if err != nil { - log.Fatalf("%s: unable to load configuration", err.Error()) + log.Fatalf("unable to load configuration: %s", err.Error()) } // Override node url in configuration file when it's explicitly set via CLI @@ -407,7 +407,7 @@ func ensureDataDirectoryExists() { if len(Config.DataDirectory) == 0 { tmpDir, err := utils.CreateTempDir() if err != nil { - log.Fatalf("%s: unable to create temporary directory", err.Error()) + log.Fatalf("unable to create temporary directory: %s", err.Error()) } Config.DataDirectory = tmpDir diff --git a/cmd/utils_asserter_configuration.go b/cmd/utils_asserter_configuration.go index 39afbdbe..db8e4100 100644 --- a/cmd/utils_asserter_configuration.go +++ b/cmd/utils_asserter_configuration.go @@ -55,18 +55,18 @@ func runCreateConfigurationCmd(cmd *cobra.Command, args []string) error { // Initialize the fetcher's asserter _, _, fetchErr := newFetcher.InitializeAsserter(Context, Config.Network, Config.ValidationFile) if fetchErr != nil { - return fmt.Errorf("%w: failed to initialize asserter", fetchErr.Err) + return fmt.Errorf("failed to initialize asserter for fetcher: %w", fetchErr.Err) } configuration, err := newFetcher.Asserter.ClientConfiguration() if err != nil { - return fmt.Errorf("%w: unable to generate spec", err) + return fmt.Errorf("unable to generate asserter configuration: %w", err) } sortArrayFieldsOnConfiguration(configuration) if err := utils.SerializeAndWrite(args[0], configuration); err != nil { - return fmt.Errorf("%w: unable to serialize asserter configuration", err) + return fmt.Errorf("unable to serialize asserter configuration: %w", err) } color.Green("Configuration file saved!") diff --git a/cmd/utils_train_zstd.go b/cmd/utils_train_zstd.go index 5a338f00..294436f9 100644 --- a/cmd/utils_train_zstd.go +++ b/cmd/utils_train_zstd.go @@ -56,7 +56,7 @@ func runTrainZstdCmd(cmd *cobra.Command, args []string) error { dictionaryPath := path.Clean(args[2]) maxItems, err := strconv.Atoi(args[3]) if err != nil { - return fmt.Errorf("%w: unable to convert max items to integer", err) + return fmt.Errorf("unable to convert max items to integer: %w", err) } compressorEntries := []*encoder.CompressorEntry{} @@ -80,7 +80,7 @@ func runTrainZstdCmd(cmd *cobra.Command, args []string) error { compressorEntries, ) if err != nil { - return fmt.Errorf("%w: badger training failed", err) + return fmt.Errorf("badger training failed: %w", err) } color.Green("Training successful!") diff --git a/cmd/validate_asserter_config.go b/cmd/validate_asserter_config.go index ba5afe38..8fddeecd 100644 --- a/cmd/validate_asserter_config.go +++ b/cmd/validate_asserter_config.go @@ -17,14 +17,15 @@ package cmd import ( "context" "fmt" - "github.com/coinbase/rosetta-cli/pkg/errors" + "reflect" + "sort" + "strings" + + cliErrs "github.com/coinbase/rosetta-cli/pkg/errors" "github.com/coinbase/rosetta-sdk-go/asserter" "github.com/coinbase/rosetta-sdk-go/fetcher" "github.com/coinbase/rosetta-sdk-go/types" "github.com/coinbase/rosetta-sdk-go/utils" - "reflect" - "sort" - "strings" ) // Common helper across Construction and Data @@ -37,46 +38,57 @@ func validateNetworkOptionsMatchesAsserterConfiguration( ) error { var asserterConfiguration asserter.Configuration if err := utils.LoadAndParse(asserterConfigurationFile, &asserterConfiguration); err != nil { - return fmt.Errorf("%w: failure loading / parsing asserter-configuration-file", err) + return fmt.Errorf("failed to load and parse asserter configuration file %s: %w", asserterConfigurationFile, err) } resp, fetchErr := f.NetworkOptions(ctx, network, nil) if fetchErr != nil { - return fmt.Errorf("%w: failure getting /network/options", fetchErr.Err) + return fmt.Errorf("failed to get network options: %w", fetchErr.Err) } - return validateNetworkAndAsserterAllowMatch(resp.Allow, &asserterConfiguration) + err := validateNetworkAndAsserterAllowMatch(resp.Allow, &asserterConfiguration) + if err != nil { + return fmt.Errorf("failed to validate network options allowlist with asserter configuration: %w", err) + } + + return nil } func validateNetworkAndAsserterAllowMatch( networkAllow *types.Allow, asserterConfiguration *asserter.Configuration, ) error { if networkAllow == nil { - return fmt.Errorf("%w: /network/options object's Allow is nil", errors.ErrAsserterConfigError) + return cliErrs.ErrNetworkOptionsAllowlistIsNil } if asserterConfiguration == nil { - return fmt.Errorf("%w: asserter-configuration-file object is nil", errors.ErrAsserterConfigError) + return cliErrs.ErrAsserterConfigurationIsNil } if err := verifyTimestampStartIndex( networkAllow.TimestampStartIndex, asserterConfiguration.AllowedTimestampStartIndex, ); err != nil { - return err + return fmt.Errorf("failed to verify timestamp start index: %w", err) } if err := verifyOperationTypes( networkAllow.OperationTypes, asserterConfiguration.AllowedOperationTypes, ); err != nil { - return err + return fmt.Errorf("failed to verify operation types: %w", err) } if err := verifyOperationStatuses( networkAllow.OperationStatuses, asserterConfiguration.AllowedOperationStatuses, ); err != nil { - return err + return fmt.Errorf("failed to verify operation statuses: %w", err) } - return verifyErrors(networkAllow.Errors, asserterConfiguration.AllowedErrors) + if err := verifyErrors( + networkAllow.Errors, asserterConfiguration.AllowedErrors, + ); err != nil { + return fmt.Errorf("failed to verify errors: %w", err) + } + + return nil } func verifyTimestampStartIndex(networkTsi *int64, assertTsi int64) error { @@ -85,10 +97,7 @@ func verifyTimestampStartIndex(networkTsi *int64, assertTsi int64) error { networkTsiVal = *networkTsi } if networkTsiVal != assertTsi { - return fmt.Errorf( - "/network/options / asserter-configuration-file timestamp start index mismatch. %d %d", - networkTsiVal, assertTsi, - ) + return fmt.Errorf("network options timestamp start index %d, asserter configuration timestamp start index %d: %w", networkTsiVal, assertTsi, cliErrs.ErrTimestampStartIndexMismatch) } return nil @@ -96,11 +105,7 @@ func verifyTimestampStartIndex(networkTsi *int64, assertTsi int64) error { func verifyOperationTypes(networkOt, asserterOt []string) error { if len(networkOt) != len(asserterOt) { - return fmt.Errorf( - "/network/options / asserter-configuration-file operation types length mismatch %+v "+ - "%+v", - networkOt, asserterOt, - ) + return fmt.Errorf("network options operation type length %d, asserter configuration operation type length %d: %w", len(networkOt), len(asserterOt), cliErrs.ErrOperationTypeLengthMismatch) } sort.Strings(networkOt) @@ -109,11 +114,7 @@ func verifyOperationTypes(networkOt, asserterOt []string) error { for i, networkOperationType := range networkOt { asserterOperationType := asserterOt[i] if networkOperationType != asserterOperationType { - return fmt.Errorf( - "/network/options / asserter-configuration-file operation type mismatch %+v "+ - "%+v\nnetwork operation types: %+v\nasserter operation types: %+v", - networkOperationType, asserterOperationType, networkOt, asserterOt, - ) + return fmt.Errorf("network options operation type %s, asserter configuration operation type %s: %w", networkOperationType, asserterOperationType, cliErrs.ErrOperationTypeMismatch) } } @@ -122,11 +123,7 @@ func verifyOperationTypes(networkOt, asserterOt []string) error { func verifyOperationStatuses(networkOs, asserterOs []*types.OperationStatus) error { if len(networkOs) != len(asserterOs) { - return fmt.Errorf( - "/network/options / asserter-configuration-file operation statuses length mismatch "+ - "%+v %+v", - networkOs, asserterOs, - ) + return fmt.Errorf("network options operation status length %d, asserter configuration operation status length %d: %w", len(networkOs), len(asserterOs), cliErrs.ErrOperationStatusLengthMismatch) } sort.Slice(networkOs, func(i, j int) bool { @@ -139,11 +136,7 @@ func verifyOperationStatuses(networkOs, asserterOs []*types.OperationStatus) err for i, networkOperationStatus := range networkOs { asserterOperationStatus := asserterOs[i] if !reflect.DeepEqual(networkOperationStatus, asserterOperationStatus) { - return fmt.Errorf( - "/network/options / asserter-configuration-file operation status mismatch %+v "+ - "%+v\nnetwork operation statuses: %+v\nasserter operation statuses: %+v", - networkOperationStatus, asserterOperationStatus, networkOs, asserterOs, - ) + return fmt.Errorf("network options operation type %s, asserter configuration operation type %s: %w", types.PrintStruct(networkOperationStatus), types.PrintStruct(asserterOperationStatus), cliErrs.ErrOperationStatusMismatch) } } @@ -152,10 +145,7 @@ func verifyOperationStatuses(networkOs, asserterOs []*types.OperationStatus) err func verifyErrors(networkErrors, asserterErrors []*types.Error) error { if len(networkErrors) != len(asserterErrors) { - return fmt.Errorf( - "/network/options / asserter-configuration-file errors length mismatch %+v %+v", - networkErrors, asserterErrors, - ) + return fmt.Errorf("network options error length %d, asserter configuration error length %d: %w", len(networkErrors), len(asserterErrors), cliErrs.ErrErrorLengthMismatch) } sort.Slice(networkErrors, func(i, j int) bool { @@ -168,11 +158,7 @@ func verifyErrors(networkErrors, asserterErrors []*types.Error) error { for i, networkError := range networkErrors { asserterError := asserterErrors[i] if !reflect.DeepEqual(networkError, asserterError) { - return fmt.Errorf( - "/network/options / asserter-configuration-file error mismatch %+v %+v\n"+ - "network errors: %+v\nasserter errors: %+v", - networkError, asserterError, networkErrors, asserterErrors, - ) + return fmt.Errorf("network options error %s, asserter configuration error %s: %w", types.PrintStruct(networkError), types.PrintStruct(asserterError), cliErrs.ErrErrorMismatch) } } diff --git a/cmd/view_balance.go b/cmd/view_balance.go index bfe3bbe2..72642452 100644 --- a/cmd/view_balance.go +++ b/cmd/view_balance.go @@ -48,11 +48,11 @@ address to specified as JSON allows for querying by SubAccountIdentifier.`, func runViewBalanceCmd(cmd *cobra.Command, args []string) error { account := &types.AccountIdentifier{} if err := json.Unmarshal([]byte(args[0]), account); err != nil { - return fmt.Errorf("%w: unable to unmarshal account %s", err, args[0]) + return fmt.Errorf("unable to unmarshal account %s: %w", args[0], err) } if err := asserter.AccountIdentifier(account); err != nil { - return fmt.Errorf("%w: invalid account identifier %s", err, types.PrintStruct(account)) + return fmt.Errorf("invalid account identifier %s: %w", types.PrintStruct(account), err) } // Create a new fetcher @@ -74,19 +74,19 @@ func runViewBalanceCmd(cmd *cobra.Command, args []string) error { // Initialize the fetcher's asserter _, _, fetchErr := newFetcher.InitializeAsserter(Context, Config.Network, Config.ValidationFile) if fetchErr != nil { - return fmt.Errorf("%w: unable to initialize asserter", fetchErr.Err) + return fmt.Errorf("unable to initialize asserter for fetcher: %w", fetchErr.Err) } _, err := utils.CheckNetworkSupported(Context, Config.Network, newFetcher) if err != nil { - return fmt.Errorf("%w: unable to confirm network is supported", err) + return fmt.Errorf("unable to confirm network %s is supported: %w", types.PrintStruct(Config.Network), err) } var lookupBlock *types.PartialBlockIdentifier if len(args) > 1 { index, err := strconv.ParseInt(args[1], 10, 64) if err != nil { - return fmt.Errorf("%w: unable to parse index %s", err, args[0]) + return fmt.Errorf("unable to parse index %s: %w", args[0], err) } lookupBlock = &types.PartialBlockIdentifier{Index: &index} @@ -100,7 +100,7 @@ func runViewBalanceCmd(cmd *cobra.Command, args []string) error { nil, ) if fetchErr != nil { - return fmt.Errorf("%w: unable to fetch account %+v", fetchErr.Err, account) + return fmt.Errorf("unable to fetch account balance for account %s: %w", types.PrintStruct(account), fetchErr.Err) } log.Printf("Amounts: %s\n", types.PrettyPrintStruct(amounts)) diff --git a/cmd/view_block.go b/cmd/view_block.go index be2de146..57834c46 100644 --- a/cmd/view_block.go +++ b/cmd/view_block.go @@ -16,10 +16,11 @@ package cmd import ( "fmt" - "github.com/coinbase/rosetta-cli/pkg/errors" "strconv" "time" + cliErrs "github.com/coinbase/rosetta-cli/pkg/errors" + "github.com/coinbase/rosetta-sdk-go/fetcher" "github.com/coinbase/rosetta-sdk-go/parser" "github.com/coinbase/rosetta-sdk-go/types" @@ -50,7 +51,7 @@ func printChanges(balanceChanges []*parser.BalanceChange) error { for _, balanceChange := range balanceChanges { parsedDiff, err := types.BigInt(balanceChange.Difference) if err != nil { - return fmt.Errorf("%w: unable to parse Difference", err) + return fmt.Errorf("unable to parse balance change difference: %w", err) } if parsedDiff.Sign() == 0 { @@ -70,7 +71,7 @@ func printChanges(balanceChanges []*parser.BalanceChange) error { func runViewBlockCmd(_ *cobra.Command, args []string) error { index, err := strconv.ParseInt(args[0], 10, 64) if err != nil { - return fmt.Errorf("%w: unable to parse index %s", err, args[0]) + return fmt.Errorf("unable to parse index %s: %w", args[0], err) } // Create a new fetcher @@ -96,12 +97,12 @@ func runViewBlockCmd(_ *cobra.Command, args []string) error { // the asserter what are valid responses. _, _, fetchErr := newFetcher.InitializeAsserter(Context, Config.Network, Config.ValidationFile) if fetchErr != nil { - return fmt.Errorf("%w: unable to initialize asserter", fetchErr.Err) + return fmt.Errorf("unable to initialize asserter for fetcher: %w", fetchErr.Err) } _, err = utils.CheckNetworkSupported(Context, Config.Network, newFetcher) if err != nil { - return fmt.Errorf("%w: unable to confirm network is supported", err) + return fmt.Errorf("unable to confirm network %s is supported: %w", types.PrintStruct(Config.Network), err) } // Fetch the specified block with retries (automatically @@ -121,11 +122,11 @@ func runViewBlockCmd(_ *cobra.Command, args []string) error { }, ) if fetchErr != nil { - return fmt.Errorf("%w: unable to fetch block", fetchErr.Err) + return fmt.Errorf("unable to fetch block %d: %w", index, fetchErr.Err) } // It's valid for a block to be omitted without triggering an error if block == nil { - return fmt.Errorf("%w: block not found, it might be omitted", errors.ErrBlockNotFound) + return cliErrs.ErrBlockNotFound } fmt.Printf("\n") @@ -140,10 +141,10 @@ func runViewBlockCmd(_ *cobra.Command, args []string) error { p := parser.New(newFetcher.Asserter, func(*types.Operation) bool { return false }, nil) balanceChanges, err := p.BalanceChanges(Context, block, false) if err != nil { - return fmt.Errorf("%w: unable to calculate balance changes", err) + return fmt.Errorf("unable to calculate balance changes: %w", err) } - fmt.Println("Cummulative:", block.BlockIdentifier.Hash) + fmt.Println("Cumulative:", block.BlockIdentifier.Hash) if err := printChanges(balanceChanges); err != nil { return err @@ -162,7 +163,7 @@ func runViewBlockCmd(_ *cobra.Command, args []string) error { }, }, false) if err != nil { - return fmt.Errorf("%w: unable to calculate balance changes", err) + return fmt.Errorf("unable to calculate balance changes: %w", err) } fmt.Println("Transaction:", tx.TransactionIdentifier.Hash) diff --git a/cmd/view_networks.go b/cmd/view_networks.go index c2d0a478..375e4db3 100644 --- a/cmd/view_networks.go +++ b/cmd/view_networks.go @@ -16,10 +16,11 @@ package cmd import ( "fmt" - "github.com/coinbase/rosetta-cli/pkg/errors" "log" "time" + "github.com/coinbase/rosetta-cli/pkg/errors" + "github.com/coinbase/rosetta-sdk-go/fetcher" "github.com/coinbase/rosetta-sdk-go/types" "github.com/fatih/color" @@ -59,7 +60,7 @@ func runViewNetworksCmd(cmd *cobra.Command, args []string) error { // Attempt to fetch network list networkList, fetchErr := f.NetworkListRetry(Context, nil) if fetchErr != nil { - return fmt.Errorf("%w: unable to fetch network list", fetchErr.Err) + return fmt.Errorf("unable to get network list: %w", fetchErr.Err) } if len(networkList.NetworkIdentifiers) == 0 { @@ -74,7 +75,7 @@ func runViewNetworksCmd(cmd *cobra.Command, args []string) error { nil, ) if fetchErr != nil { - return fmt.Errorf("%w: unable to get network options", fetchErr.Err) + return fmt.Errorf("unable to get network options: %w", fetchErr.Err) } log.Printf("Network options: %s\n", types.PrettyPrintStruct(networkOptions)) @@ -85,7 +86,7 @@ func runViewNetworksCmd(cmd *cobra.Command, args []string) error { nil, ) if fetchErr != nil { - return fmt.Errorf("%w: unable to get network status", fetchErr.Err) + return fmt.Errorf("unable to get network status: %w", fetchErr.Err) } log.Printf("Network status: %s\n", types.PrettyPrintStruct(networkStatus)) diff --git a/pkg/errors/errors.go b/pkg/errors/errors.go index 1b608be2..02e34a5b 100644 --- a/pkg/errors/errors.go +++ b/pkg/errors/errors.go @@ -40,14 +40,28 @@ var ( // Data check errors ErrDataCheckHalt = errors.New("data check halted") - ErrInitDataTester = errors.New("unexpected error occurred while trying to initialize data tester") ErrReconciliationFailure = errors.New("reconciliation failure") + // Spec check errors + ErrErrorEmptyMessage = errors.New("error object can't have empty message") + ErrErrorNegativeCode = errors.New("error object can't have negative code") + ErrAccountNullPointer = errors.New("account is nil") + ErrBlockNotIdempotent = errors.New("multiple calls with the same hash don't return the same block") + ErrBlockTip = errors.New("unspecified block_identifier doesn't give the tip block") + // Construction check errors ErrConstructionCheckHalt = errors.New("construction check halted") // Command errors - ErrBlockNotFound = errors.New("block not found") - ErrNoAvailableNetwork = errors.New("no networks available") - ErrAsserterConfigError = errors.New("asserter configuration validation failed") + ErrBlockNotFound = errors.New("block not found") + ErrNoAvailableNetwork = errors.New("no networks available") + ErrNetworkOptionsAllowlistIsNil = errors.New("network options allowlist is nil") + ErrAsserterConfigurationIsNil = errors.New("asserter configuration is nil") + ErrTimestampStartIndexMismatch = errors.New("timestamp start index mismatch") + ErrOperationTypeLengthMismatch = errors.New("operation type length mismatch") + ErrOperationTypeMismatch = errors.New("operation type mismatch") + ErrOperationStatusLengthMismatch = errors.New("operation status length mismatch") + ErrOperationStatusMismatch = errors.New("operation status mismatch") + ErrErrorLengthMismatch = errors.New("error length mismatch") + ErrErrorMismatch = errors.New("error mismatch") ) diff --git a/pkg/logger/logger.go b/pkg/logger/logger.go index 9fd9cce2..2bca8eda 100644 --- a/pkg/logger/logger.go +++ b/pkg/logger/logger.go @@ -98,7 +98,7 @@ func NewLogger( ) (*Logger, error) { zapLogger, err := buildZapLogger(checkType, network, fields...) if err != nil { - return nil, err + return nil, fmt.Errorf("failed to build zap logger: %w", err) } return &Logger{ logDir: logDir, @@ -118,7 +118,7 @@ func buildZapLogger( config := zap.NewDevelopmentConfig() config.EncoderConfig.EncodeLevel = zapcore.CapitalColorLevelEncoder - baseSlice := []zap.Field { + baseSlice := []zap.Field{ zap.String("blockchain", network.Blockchain), zap.String("network", network.Network), zap.String("check_type", string(checkType)), @@ -236,7 +236,7 @@ func (l *Logger) AddBlockStream( os.FileMode(utils.DefaultFilePermissions), ) if err != nil { - return err + return fmt.Errorf("failed to open file %s: %w", path.Join(l.logDir, blockStreamFile), err) } defer closeFile(f) @@ -251,7 +251,7 @@ func (l *Logger) AddBlockStream( ) fmt.Print(blockString) if _, err := f.WriteString(blockString); err != nil { - return err + return fmt.Errorf("failed to write block string %s: %w", blockString, err) } return l.TransactionStream(ctx, block) @@ -273,7 +273,7 @@ func (l *Logger) RemoveBlockStream( os.FileMode(utils.DefaultFilePermissions), ) if err != nil { - return err + return fmt.Errorf("failed to open file %s: %w", path.Join(l.logDir, blockStreamFile), err) } defer closeFile(f) @@ -286,7 +286,7 @@ func (l *Logger) RemoveBlockStream( ) fmt.Print(blockString) _, err = f.WriteString(blockString) - return err + return fmt.Errorf("failed to write block string %s: %w", blockString, err) } // TransactionStream writes the next processed block's transactions @@ -305,7 +305,7 @@ func (l *Logger) TransactionStream( os.FileMode(utils.DefaultFilePermissions), ) if err != nil { - return err + return fmt.Errorf("failed to open file %s: %w", path.Join(l.logDir, transactionStreamFile), err) } defer closeFile(f) @@ -317,12 +317,10 @@ func (l *Logger) TransactionStream( block.BlockIdentifier.Index, block.BlockIdentifier.Hash, ) - fmt.Print(transactionString) _, err = f.WriteString(transactionString) - if err != nil { - return err + return fmt.Errorf("failed to write transaction string %s: %w", transactionString, err) } for _, op := range tx.Operations { @@ -342,7 +340,7 @@ func (l *Logger) TransactionStream( networkIndex = *op.OperationIdentifier.NetworkIndex } - _, err = f.WriteString(fmt.Sprintf( + transactionOperationString := fmt.Sprintf( "TxOp %d(%d) %s %s %s %s %s\n", op.OperationIdentifier.Index, networkIndex, @@ -351,9 +349,10 @@ func (l *Logger) TransactionStream( amount, symbol, *op.Status, - )) + ) + _, err = f.WriteString(transactionOperationString) if err != nil { - return err + return fmt.Errorf("failed to write transaction operation string %s: %w", transactionOperationString, err) } } } @@ -377,7 +376,7 @@ func (l *Logger) BalanceStream( os.FileMode(utils.DefaultFilePermissions), ) if err != nil { - return err + return fmt.Errorf("failed to open file %s: %w", path.Join(l.logDir, balanceStreamFile), err) } defer closeFile(f) @@ -393,7 +392,7 @@ func (l *Logger) BalanceStream( ) if _, err := f.WriteString(fmt.Sprintf("%s\n", balanceLog)); err != nil { - return err + return fmt.Errorf("failed to write balance log %s: %w", balanceLog, err) } } return nil @@ -419,7 +418,7 @@ func (l *Logger) ReconcileSuccessStream( os.FileMode(utils.DefaultFilePermissions), ) if err != nil { - return err + return fmt.Errorf("failed to open file %s: %w", path.Join(l.logDir, reconcileSuccessStreamFile), err) } defer closeFile(f) @@ -431,7 +430,7 @@ func (l *Logger) ReconcileSuccessStream( block.Index, ) - _, err = f.WriteString(fmt.Sprintf( + reconciliationSuccessString := fmt.Sprintf( "Type:%s Account: %s Currency: %s Balance: %s Block: %d:%s\n", reconciliationType, types.AccountString(account), @@ -439,9 +438,10 @@ func (l *Logger) ReconcileSuccessStream( balance, block.Index, block.Hash, - )) + ) + _, err = f.WriteString(reconciliationSuccessString) if err != nil { - return err + return fmt.Errorf("failed to write reconciliation success string %s: %w", reconciliationSuccessString, err) } return nil @@ -490,12 +490,12 @@ func (l *Logger) ReconcileFailureStream( os.FileMode(utils.DefaultFilePermissions), ) if err != nil { - return err + return fmt.Errorf("failed to open file %s: %w", path.Join(l.logDir, reconcileFailureStreamFile), err) } defer closeFile(f) - _, err = f.WriteString(fmt.Sprintf( + reconciliationFailureString := fmt.Sprintf( "Type:%s Account: %s Currency: %s Block: %s:%d computed: %s live: %s\n", reconciliationType, types.AccountString(account), @@ -504,9 +504,10 @@ func (l *Logger) ReconcileFailureStream( block.Index, computedBalance, liveBalance, - )) + ) + _, err = f.WriteString(reconciliationFailureString) if err != nil { - return err + return fmt.Errorf("failed to write reconciliation failure string %s: %w", reconciliationFailureString, err) } return nil @@ -546,7 +547,7 @@ func (l *Logger) Fatal(msg string, fields ...zap.Field) { func closeFile(f *os.File) { err := f.Close() if err != nil { - log.Fatal(fmt.Errorf("%w: unable to close file", err)) + log.Fatal(fmt.Errorf("unable to close file: %w", err)) } } From f957616f5188829fe57ea36fe42a664374a3a748 Mon Sep 17 00:00:00 2001 From: Jingfu Wang Date: Tue, 13 Sep 2022 14:56:27 -0400 Subject: [PATCH 351/376] feat: error handling for processor and tester Signed-off-by: Jingfu Wang --- pkg/errors/errors.go | 37 +++++---- pkg/processor/balance_storage_handler.go | 5 +- pkg/processor/balance_storage_helper.go | 2 +- pkg/processor/broadcast_storage_handler.go | 8 +- pkg/processor/broadcast_storage_helper.go | 9 ++- pkg/processor/coordinator_helper.go | 22 ++--- pkg/processor/reconciler_handler.go | 16 ++-- pkg/processor/reconciler_helper.go | 3 +- pkg/results/construction_results.go | 2 +- pkg/results/data_results.go | 36 ++++----- pkg/results/perf_results.go | 2 +- pkg/tester/construction.go | 43 +++++----- pkg/tester/data.go | 93 +++++++++++----------- pkg/tester/data_perf.go | 6 +- 14 files changed, 149 insertions(+), 135 deletions(-) diff --git a/pkg/errors/errors.go b/pkg/errors/errors.go index 02e34a5b..5fa4293e 100644 --- a/pkg/errors/errors.go +++ b/pkg/errors/errors.go @@ -39,8 +39,12 @@ var ( ErrConstructionConfigMissing = errors.New("construction configuration is missing") // Data check errors - ErrDataCheckHalt = errors.New("data check halted") - ErrReconciliationFailure = errors.New("reconciliation failure") + ErrDataCheckHalt = errors.New("data check halted") + ErrReconciliationFailure = errors.New("reconciliation failure") + ErrInitDataTester = errors.New("unexpected error occurred while trying to initialize data tester") + ErrReconcilerDrainHalt = errors.New("reconciler queue drain halted") + ErrMissingOps = errors.New("search for block with missing ops halted") + ErrUnableToFindMissingOps = errors.New("unable to find missing ops") // Spec check errors ErrErrorEmptyMessage = errors.New("error object can't have empty message") @@ -50,18 +54,23 @@ var ( ErrBlockTip = errors.New("unspecified block_identifier doesn't give the tip block") // Construction check errors - ErrConstructionCheckHalt = errors.New("construction check halted") + ErrConstructionCheckHalt = errors.New("construction check halted") + ErrBalanceExemptionsWithInitialBalanceFetchDisabled = errors.New("found balance exemptions but initial balance fetch disabled") // Command errors - ErrBlockNotFound = errors.New("block not found") - ErrNoAvailableNetwork = errors.New("no networks available") - ErrNetworkOptionsAllowlistIsNil = errors.New("network options allowlist is nil") - ErrAsserterConfigurationIsNil = errors.New("asserter configuration is nil") - ErrTimestampStartIndexMismatch = errors.New("timestamp start index mismatch") - ErrOperationTypeLengthMismatch = errors.New("operation type length mismatch") - ErrOperationTypeMismatch = errors.New("operation type mismatch") - ErrOperationStatusLengthMismatch = errors.New("operation status length mismatch") - ErrOperationStatusMismatch = errors.New("operation status mismatch") - ErrErrorLengthMismatch = errors.New("error length mismatch") - ErrErrorMismatch = errors.New("error mismatch") + ErrBlockNotFound = errors.New("block not found") + ErrNoAvailableNetwork = errors.New("no networks available") + ErrNetworkOptionsAllowlistIsNil = errors.New("network options allowlist is nil") + ErrAsserterConfigurationIsNil = errors.New("asserter configuration is nil") + ErrTimestampStartIndexMismatch = errors.New("timestamp start index mismatch") + ErrOperationTypeLengthMismatch = errors.New("operation type length mismatch") + ErrOperationTypeMismatch = errors.New("operation type mismatch") + ErrOperationStatusLengthMismatch = errors.New("operation status length mismatch") + ErrOperationStatusMismatch = errors.New("operation status mismatch") + ErrErrorLengthMismatch = errors.New("error length mismatch") + ErrErrorMismatch = errors.New("error mismatch") + ErrAsserterConfigError = errors.New("asserter configuration validation failed") + ErrNoHeadBlock = errors.New("no head block") + ErrBlockBenchmarkTimeout = errors.New("/block endpoint benchmarking timed out") + ErrAccountBalanceBenchmarkTimeout = errors.New("/account/balance endpoint benchmarking timed out") ) diff --git a/pkg/processor/balance_storage_handler.go b/pkg/processor/balance_storage_handler.go index 84e0df18..9dd251e8 100644 --- a/pkg/processor/balance_storage_handler.go +++ b/pkg/processor/balance_storage_handler.go @@ -16,6 +16,7 @@ package processor import ( "context" + "fmt" "math/big" "github.com/coinbase/rosetta-cli/pkg/logger" @@ -124,7 +125,7 @@ func (h *BalanceStorageHandler) AccountsReconciled( modules.ReconciledAccounts, big.NewInt(int64(count)), ) - return err + return fmt.Errorf("failed to update the total accounts reconciled by count: %w", err) } // AccountsSeen updates the total accounts seen by count. @@ -139,5 +140,5 @@ func (h *BalanceStorageHandler) AccountsSeen( modules.SeenAccounts, big.NewInt(int64(count)), ) - return err + return fmt.Errorf("failed to update the total accounts seen by count: %w", err) } diff --git a/pkg/processor/balance_storage_helper.go b/pkg/processor/balance_storage_helper.go index f862093f..f043691d 100644 --- a/pkg/processor/balance_storage_helper.go +++ b/pkg/processor/balance_storage_helper.go @@ -110,7 +110,7 @@ func (h *BalanceStorageHelper) AccountBalance( lookupBlock.Index, ) if err != nil { - return nil, fmt.Errorf("%w: unable to get currency balance", err) + return nil, fmt.Errorf("unable to get balance of currency %s for account %s: %w", types.PrintStruct(currency), types.PrintStruct(account), err) } // If the returned balance block does not match the intended diff --git a/pkg/processor/broadcast_storage_handler.go b/pkg/processor/broadcast_storage_handler.go index edb57c1e..aad34491 100644 --- a/pkg/processor/broadcast_storage_handler.go +++ b/pkg/processor/broadcast_storage_handler.go @@ -72,7 +72,7 @@ func (h *BroadcastStorageHandler) TransactionConfirmed( ) error { _, _, relatedTransactions, err := h.blockStorage.FindRelatedTransactions(ctx, transaction.TransactionIdentifier, dbTx) if err != nil { - return fmt.Errorf("%w: could not find related transactions", err) + return fmt.Errorf("failed to find related transactions %s: %w", types.PrintStruct(transaction.TransactionIdentifier), err) } observed := transaction.Operations @@ -81,7 +81,7 @@ func (h *BroadcastStorageHandler) TransactionConfirmed( } if err := h.parser.ExpectedOperations(intent, observed, false, true); err != nil { - return fmt.Errorf("%w: confirmed transaction did not match intent", err) + return fmt.Errorf("confirmed transaction did not match intent: %w", err) } // Validate destination memo if it's needed @@ -108,7 +108,7 @@ func (h *BroadcastStorageHandler) TransactionConfirmed( identifier, transaction, ); err != nil { - return fmt.Errorf("%w: coordinator could not handle transaction", err) + return fmt.Errorf("coordinator could not handle transaction: %w", err) } return nil @@ -155,7 +155,7 @@ func (h *BroadcastStorageHandler) BroadcastFailed( identifier, nil, ); err != nil { - return fmt.Errorf("%w: coordinator could not handle transaction", err) + return fmt.Errorf("coordinator could not handle transaction: %w", err) } if h.config.Construction.IgnoreBroadcastFailures { diff --git a/pkg/processor/broadcast_storage_helper.go b/pkg/processor/broadcast_storage_helper.go index 5e881852..c5095599 100644 --- a/pkg/processor/broadcast_storage_helper.go +++ b/pkg/processor/broadcast_storage_helper.go @@ -17,6 +17,7 @@ package processor import ( "context" "fmt" + "github.com/coinbase/rosetta-sdk-go/utils" "github.com/coinbase/rosetta-sdk-go/fetcher" @@ -55,7 +56,7 @@ func (h *BroadcastStorageHelper) AtTip( ) (bool, error) { atTip, _, err := utils.CheckStorageTip(ctx, h.network, tipDelay, h.fetcher, h.blockStorage) if err != nil { - return false, err + return false, fmt.Errorf("failed to check storage tip: %w", err) } return atTip, nil @@ -68,7 +69,7 @@ func (h *BroadcastStorageHelper) CurrentBlockIdentifier( ) (*types.BlockIdentifier, error) { blockIdentifier, err := h.blockStorage.GetHeadBlockIdentifier(ctx) if err != nil { - return nil, fmt.Errorf("%w: unable to get head block identifier", err) + return nil, fmt.Errorf("unable to get head block identifier: %w", err) } return blockIdentifier, nil @@ -84,7 +85,7 @@ func (h *BroadcastStorageHelper) FindTransaction( ) (*types.BlockIdentifier, *types.Transaction, error) { newestBlock, transaction, err := h.blockStorage.FindTransaction(ctx, transactionIdentifier, txn) if err != nil { - return nil, nil, fmt.Errorf("%w: unable to perform transaction search", err) + return nil, nil, fmt.Errorf("unable to perform transaction search for transaction %s: %w", types.PrintStruct(transactionIdentifier), err) } return newestBlock, transaction, nil @@ -103,7 +104,7 @@ func (h *BroadcastStorageHelper) BroadcastTransaction( networkTransaction, ) if fetchErr != nil { - return nil, fmt.Errorf("%w: unable to broadcast transaction", fetchErr.Err) + return nil, fmt.Errorf("unable to broadcast transaction %s: %w", networkTransaction, fetchErr.Err) } return transactionIdentifier, nil diff --git a/pkg/processor/coordinator_helper.go b/pkg/processor/coordinator_helper.go index 496611bd..f21d91aa 100644 --- a/pkg/processor/coordinator_helper.go +++ b/pkg/processor/coordinator_helper.go @@ -16,11 +16,11 @@ package processor import ( "context" - "errors" "fmt" "log" "math/big" + cliErrs "github.com/coinbase/rosetta-cli/pkg/errors" "github.com/coinbase/rosetta-sdk-go/constructor/coordinator" "github.com/coinbase/rosetta-sdk-go/fetcher" "github.com/coinbase/rosetta-sdk-go/keys" @@ -153,7 +153,7 @@ func (c *CoordinatorHelper) Derive( ) if fetchErr != nil { c.verboseLog(reqerror, constructionDerive, arg{argError, fetchErr}) - return nil, nil, fetchErr.Err + return nil, nil, fmt.Errorf("/construction/derive call is failed: %w", fetchErr.Err) } c.verboseLog(response, constructionDerive, @@ -185,7 +185,7 @@ func (c *CoordinatorHelper) Preprocess( if fetchErr != nil { c.verboseLog(reqerror, constructionPreprocess, arg{argError, fetchErr}) - return nil, nil, fetchErr.Err + return nil, nil, fmt.Errorf("/construction/preprocess call is failed: %w", fetchErr.Err) } c.verboseLog(response, constructionPreprocess, @@ -217,7 +217,7 @@ func (c *CoordinatorHelper) Metadata( if fetchErr != nil { c.verboseLog(reqerror, constructionMetadata, arg{argError, fetchErr}) - return nil, nil, fetchErr.Err + return nil, nil, fmt.Errorf("/construction/metadata call is failed: %w", fetchErr.Err) } c.verboseLog(response, constructionMetadata, @@ -251,7 +251,7 @@ func (c *CoordinatorHelper) Payloads( if fetchErr != nil { c.verboseLog(reqerror, constructionPayloads, arg{argError, fetchErr}) - return "", nil, fetchErr.Err + return "", nil, fmt.Errorf("/construction/payloads call is failed: %w", fetchErr.Err) } c.verboseLog(response, constructionPayloads, @@ -283,7 +283,7 @@ func (c *CoordinatorHelper) Parse( if fetchErr != nil { c.verboseLog(reqerror, constructionParse, arg{argError, fetchErr}) - return nil, nil, nil, fetchErr.Err + return nil, nil, nil, fmt.Errorf("/construction/parse call is failed: %w", fetchErr.Err) } c.verboseLog(response, constructionParse, @@ -316,7 +316,7 @@ func (c *CoordinatorHelper) Combine( if fetchErr != nil { c.verboseLog(reqerror, constructionCombine, arg{argError, fetchErr}) - return "", fetchErr.Err + return "", fmt.Errorf("/construction/combine call is failed: %w", fetchErr.Err) } c.verboseLog(response, constructionCombine, arg{argNetworkTransaction, res}) @@ -342,7 +342,7 @@ func (c *CoordinatorHelper) Hash( if fetchErr != nil { c.verboseLog(reqerror, constructionHash, arg{argError, fetchErr}) - return nil, fetchErr.Err + return nil, fmt.Errorf("/construction/hash call is failed: %w", fetchErr.Err) } c.verboseLog(response, constructionHash, arg{argTransactionIdentifier, res}) @@ -400,10 +400,10 @@ func (c *CoordinatorHelper) Balance( ) (*types.Amount, error) { headBlock, err := c.blockStorage.GetHeadBlockIdentifier(ctx) if err != nil { - return nil, err + return nil, fmt.Errorf("failed to get head block identifier: %w", err) } if headBlock == nil { - return nil, errors.New("no blocks synced") + return nil, cliErrs.ErrNoHeadBlock } return c.balanceStorage.GetOrSetBalanceTransactional( @@ -429,7 +429,7 @@ func (c *CoordinatorHelper) Coins( accountIdentifier, ) if err != nil { - return nil, fmt.Errorf("%w: unable to get coins", err) + return nil, fmt.Errorf("unable to get coins for account %s: %w", types.PrintStruct(accountIdentifier), err) } coinsToReturn := []*types.Coin{} diff --git a/pkg/processor/reconciler_handler.go b/pkg/processor/reconciler_handler.go index f7148b74..b6a6154b 100644 --- a/pkg/processor/reconciler_handler.go +++ b/pkg/processor/reconciler_handler.go @@ -112,7 +112,7 @@ func (h *ReconcilerHandler) UpdateCounts(ctx context.Context) error { } if _, err := h.counterStorage.Update(ctx, key, big.NewInt(count)); err != nil { - return err + return fmt.Errorf("failed to key %s in counter storage: %w", key, err) } } @@ -145,7 +145,7 @@ func (h *ReconcilerHandler) ReconciliationFailed( block, ) if err != nil { - return err + return fmt.Errorf("failed to log reconciliation checks when reconciliation is failed: %w", err) } if h.haltOnReconciliationError { @@ -161,28 +161,28 @@ func (h *ReconcilerHandler) ReconciliationFailed( } h.InactiveFailureBlock = block return fmt.Errorf( - "%w: inactive reconciliation error for %s at %d (computed: %s%s, live: %s%s)", - cliErrs.ErrReconciliationFailure, + "inactive reconciliation error for account address %s at block index %d (computed: %s%s, live: %s%s): %w", account.Address, block.Index, computedBalance, currency.Symbol, liveBalance, currency.Symbol, + cliErrs.ErrReconciliationFailure, ) } // If we halt on an active reconciliation error, store in the handler. h.ActiveFailureBlock = block return fmt.Errorf( - "%w: active reconciliation error for %s at %d (computed: %s%s, live: %s%s)", - cliErrs.ErrReconciliationFailure, + "active reconciliation error for account address %s at block index %d (computed: %s%s, live: %s%s): %w", account.Address, block.Index, computedBalance, currency.Symbol, liveBalance, currency.Symbol, + cliErrs.ErrReconciliationFailure, ) } @@ -209,7 +209,7 @@ func (h *ReconcilerHandler) ReconciliationExempt( // we still mark the account as being reconciled because the balance was in the range // specified by exemption. if err := h.balanceStorage.Reconciled(ctx, account, currency, block); err != nil { - return fmt.Errorf("%w: unable to store updated reconciliation", err) + return fmt.Errorf("unable to store updated reconciliation currency %s of account %s at block %s: %w", types.PrintStruct(currency), types.PrintStruct(account), types.PrintStruct(block), err) } return nil @@ -250,7 +250,7 @@ func (h *ReconcilerHandler) ReconciliationSucceeded( h.counterLock.Unlock() if err := h.balanceStorage.Reconciled(ctx, account, currency, block); err != nil { - return fmt.Errorf("%w: unable to store updated reconciliation", err) + return fmt.Errorf("unable to store updated reconciliation currency %s of account %s at block %s: %w", types.PrintStruct(currency), types.PrintStruct(account), types.PrintStruct(block), err) } return h.logger.ReconcileSuccessStream( diff --git a/pkg/processor/reconciler_helper.go b/pkg/processor/reconciler_helper.go index 3fb163fc..dd50c760 100644 --- a/pkg/processor/reconciler_helper.go +++ b/pkg/processor/reconciler_helper.go @@ -16,6 +16,7 @@ package processor import ( "context" + "fmt" "github.com/coinbase/rosetta-cli/configuration" @@ -138,7 +139,7 @@ func (h *ReconcilerHelper) LiveBalance( index, ) if err != nil { - return nil, nil, err + return nil, nil, fmt.Errorf("failed to get current balance of currency %s of account %s: %w", types.PrintStruct(currency), types.PrintStruct(account), err) } return amt, block, nil } diff --git a/pkg/results/construction_results.go b/pkg/results/construction_results.go index e578b263..3d968247 100644 --- a/pkg/results/construction_results.go +++ b/pkg/results/construction_results.go @@ -65,7 +65,7 @@ func (c *CheckConstructionResults) Output(path string) { if len(path) > 0 { writeErr := utils.SerializeAndWrite(path, c) if writeErr != nil { - log.Printf("%s: unable to save results\n", writeErr.Error()) + log.Printf("unable to save results: %s\n", writeErr.Error()) } } } diff --git a/pkg/results/data_results.go b/pkg/results/data_results.go index 6ba9251d..7ff08dfa 100644 --- a/pkg/results/data_results.go +++ b/pkg/results/data_results.go @@ -92,7 +92,7 @@ func (c *CheckDataResults) Output(path string) { if len(path) > 0 { writeErr := utils.SerializeAndWrite(path, c) if writeErr != nil { - log.Printf("%s: unable to save results\n", writeErr.Error()) + log.Printf("unable to save results: %s\n", writeErr.Error()) } } } @@ -192,61 +192,61 @@ func ComputeCheckDataStats( blocks, err := counters.Get(ctx, modules.BlockCounter) if err != nil { - log.Printf("%s: cannot get block counter", err.Error()) + log.Printf("cannot get block counter: %s", err.Error()) return nil } orphans, err := counters.Get(ctx, modules.OrphanCounter) if err != nil { - log.Printf("%s: cannot get orphan counter", err.Error()) + log.Printf("cannot get orphan counter: %s", err.Error()) return nil } txs, err := counters.Get(ctx, modules.TransactionCounter) if err != nil { - log.Printf("%s: cannot get transaction counter", err.Error()) + log.Printf("cannot get transaction counter: %s", err.Error()) return nil } ops, err := counters.Get(ctx, modules.OperationCounter) if err != nil { - log.Printf("%s: cannot get operations counter", err.Error()) + log.Printf("cannot get operations counter: %s", err.Error()) return nil } accounts, err := counters.Get(ctx, modules.SeenAccounts) if err != nil { - log.Printf("%s: cannot get accounts counter", err.Error()) + log.Printf("cannot get accounts counter: %s", err.Error()) return nil } activeReconciliations, err := counters.Get(ctx, modules.ActiveReconciliationCounter) if err != nil { - log.Printf("%s: cannot get active reconciliations counter", err.Error()) + log.Printf("cannot get active reconciliations counter: %s", err.Error()) return nil } inactiveReconciliations, err := counters.Get(ctx, modules.InactiveReconciliationCounter) if err != nil { - log.Printf("%s: cannot get inactive reconciliations counter", err.Error()) + log.Printf("cannot get inactive reconciliations counter: %s", err.Error()) return nil } exemptReconciliations, err := counters.Get(ctx, modules.ExemptReconciliationCounter) if err != nil { - log.Printf("%s: cannot get exempt reconciliations counter", err.Error()) + log.Printf("cannot get exempt reconciliations counter: %s", err.Error()) return nil } failedReconciliations, err := counters.Get(ctx, modules.FailedReconciliationCounter) if err != nil { - log.Printf("%s: cannot get failed reconciliations counter", err.Error()) + log.Printf("cannot get failed reconciliations counter: %s", err.Error()) return nil } skippedReconciliations, err := counters.Get(ctx, modules.SkippedReconciliationsCounter) if err != nil { - log.Printf("%s: cannot get skipped reconciliations counter", err.Error()) + log.Printf("cannot get skipped reconciliations counter: %s", err.Error()) return nil } @@ -272,7 +272,7 @@ func ComputeCheckDataStats( // In this case, we use the default 0 value for the reconciliation // coverage in stats. case err != nil: - log.Printf("%s: cannot get reconciliation coverage", err.Error()) + log.Printf("cannot get reconciliation coverage: %s", err.Error()) return nil } } @@ -304,7 +304,7 @@ func ComputeCheckDataProgress( ) *CheckDataProgress { networkStatus, fetchErr := fetcher.NetworkStatusRetry(ctx, network, nil) if fetchErr != nil { - fmt.Printf("%s: cannot get network status", fetchErr.Err.Error()) + fmt.Printf("cannot get network status: %s", fetchErr.Err.Error()) return nil } tipIndex := networkStatus.CurrentBlockIdentifier.Index @@ -315,19 +315,19 @@ func ComputeCheckDataProgress( } // Get current tip in the case that re-orgs occurred - // or a custom start index was provied. + // or a custom start index was provided. headBlock, err := blockStorage.GetHeadBlockIdentifier(ctx) if errors.Is(err, storageErrs.ErrHeadBlockNotFound) { return nil } if err != nil { - fmt.Printf("%s: cannot get head block", err.Error()) + fmt.Printf("cannot get head block: %s", err.Error()) return nil } blocks, err := counters.Get(ctx, modules.BlockCounter) if err != nil { - fmt.Printf("%s: cannot get block counter", err.Error()) + fmt.Printf("cannot get block counter: %s", err.Error()) return nil } @@ -337,7 +337,7 @@ func ComputeCheckDataProgress( orphans, err := counters.Get(ctx, modules.OrphanCounter) if err != nil { - fmt.Printf("%s: cannot get orphan counter", err.Error()) + fmt.Printf("cannot get orphan counter: %s", err.Error()) return nil } @@ -350,7 +350,7 @@ func ComputeCheckDataProgress( elapsedTime, err := counters.Get(ctx, TimeElapsedCounter) if err != nil { - fmt.Printf("%s: cannot get elapsed time", err.Error()) + fmt.Printf("cannot get elapsed time: %s", err.Error()) return nil } diff --git a/pkg/results/perf_results.go b/pkg/results/perf_results.go index a0349c6e..9d5c7ee2 100644 --- a/pkg/results/perf_results.go +++ b/pkg/results/perf_results.go @@ -32,7 +32,7 @@ func (c *CheckPerfStats) Output(path string) { if len(path) > 0 { writeErr := utils.SerializeAndWrite(path, c) if writeErr != nil { - log.Printf("%s: unable to save results\n", writeErr.Error()) + log.Printf("unable to save results: %s\n", writeErr.Error()) } } } diff --git a/pkg/tester/construction.go b/pkg/tester/construction.go index 537890ca..e6d8d466 100644 --- a/pkg/tester/construction.go +++ b/pkg/tester/construction.go @@ -19,11 +19,12 @@ import ( "encoding/json" "errors" "fmt" - customErrs "github.com/coinbase/rosetta-cli/pkg/errors" "log" "net/http" "time" + cliErrs "github.com/coinbase/rosetta-cli/pkg/errors" + "github.com/coinbase/rosetta-cli/configuration" "github.com/coinbase/rosetta-cli/pkg/logger" "github.com/coinbase/rosetta-cli/pkg/processor" @@ -84,7 +85,7 @@ func InitializeConstruction( ) (*ConstructionTester, error) { dataPath, err := utils.CreateCommandPath(config.DataDirectory, constructionCmdName, network) if err != nil { - return nil, fmt.Errorf("%s: cannot create command path", err.Error()) + return nil, fmt.Errorf("failed to create command path: %w", err) } opts := []database.BadgerOption{} @@ -100,17 +101,17 @@ func InitializeConstruction( localStore, err := database.NewBadgerDatabase(ctx, dataPath, opts...) if err != nil { - return nil, fmt.Errorf("%s: unable to initialize database", err.Error()) + return nil, fmt.Errorf("unable to initialize database: %w", err) } networkOptions, fetchErr := onlineFetcher.NetworkOptionsRetry(ctx, network, nil) if err != nil { - return nil, fmt.Errorf("%s: unable to get network options", fetchErr.Err.Error()) + return nil, fmt.Errorf("unable to get network options: %w", fetchErr.Err) } if len(networkOptions.Allow.BalanceExemptions) > 0 && config.Construction.InitialBalanceFetchDisabled { - return nil, fmt.Errorf("found balance exemptions but initial balance fetch disabled") + return nil, cliErrs.ErrBalanceExemptionsWithInitialBalanceFetchDisabled } counterStorage := modules.NewCounterStorage(localStore) @@ -124,7 +125,7 @@ func InitializeConstruction( network, ) if err != nil { - return nil, fmt.Errorf("unable to initialize logger with error: %s", err.Error()) + return nil, fmt.Errorf("unable to initialize logger with error: %w", err) } blockStorage := modules.NewBlockStorage(localStore, config.SerialBlockWorkers) @@ -194,7 +195,7 @@ func InitializeConstruction( // Load all accounts for network accounts, err := keyStorage.GetAllAccounts(ctx) if err != nil { - return nil, fmt.Errorf("%w: unable to load addresses", err) + return nil, fmt.Errorf("unable to load addresses: %w", err) } // Track balances on all addresses @@ -230,12 +231,12 @@ func InitializeConstruction( accBalances, err := utils.GetAccountBalances(ctx, onlineFetcher, accountBalanceRequests) if err != nil { - return nil, fmt.Errorf("%w: unable to get account balances", err) + return nil, fmt.Errorf("unable to get account balances: %w", err) } err = balanceStorage.SetBalanceImported(ctx, nil, accBalances) if err != nil { - return nil, fmt.Errorf("%w: unable to set balances", err) + return nil, fmt.Errorf("unable to set balances: %w", err) } // ------------------------------------------------------------------------- @@ -245,7 +246,7 @@ func InitializeConstruction( if config.CoinSupported { acctCoins, errAccCoins := utils.GetAccountCoins(ctx, onlineFetcher, acctCoinsReqs) if errAccCoins != nil { - return nil, fmt.Errorf("%w: unable to get account coins", errAccCoins) + return nil, fmt.Errorf("unable to get account coins: %w", errAccCoins) } // Extract accounts from account coins requests @@ -256,7 +257,7 @@ func InitializeConstruction( err = coinStorage.SetCoinsImported(ctx, accts, acctCoins) if err != nil { - return nil, fmt.Errorf("%w: unable to set coin balances", err) + return nil, fmt.Errorf("unable to set coin balances: %w", err) } } @@ -290,7 +291,7 @@ func InitializeConstruction( config.Construction.Workflows, ) if err != nil { - log.Fatalf("%s: unable to create coordinator", err.Error()) + log.Fatalf("unable to create coordinator: %s", err.Error()) } broadcastHandler := processor.NewBroadcastStorageHandler( @@ -338,7 +339,7 @@ func InitializeConstruction( // CloseDatabase closes the database used by ConstructionTester. func (t *ConstructionTester) CloseDatabase(ctx context.Context) { if err := t.database.Close(ctx); err != nil { - log.Fatalf("%s: error closing database", err.Error()) + log.Fatalf("error closing database: %s", err.Error()) } } @@ -375,7 +376,7 @@ func (t *ConstructionTester) checkTip(ctx context.Context) (int64, error) { t.onlineFetcher, ) if err != nil { - return -1, err + return -1, fmt.Errorf("failed to check network tip: %w", err) } if atTip { @@ -394,7 +395,7 @@ func (t *ConstructionTester) waitForTip(ctx context.Context) (int64, error) { // Don't wait any time before first tick if at tip. tipIndex, err := t.checkTip(ctx) if err != nil { - return -1, err + return -1, fmt.Errorf("failed to check tip: %w", err) } if tipIndex != -1 { @@ -426,10 +427,10 @@ func (t *ConstructionTester) StartSyncer( // we will unnecessarily sync tons of blocks before reaching any that matter. startIndex, err = t.waitForTip(ctx) if err != nil { - return fmt.Errorf("%w: unable to wait for tip", err) + return fmt.Errorf("unable to wait for tip: %w", err) } } else if err != nil { - return fmt.Errorf("%w: unable to get last block synced", err) + return fmt.Errorf("unable to get last block synced: %w", err) } return t.syncer.Sync(ctx, startIndex, -1) @@ -444,7 +445,7 @@ func (t *ConstructionTester) StartConstructor( if t.config.Construction.ClearBroadcasts { broadcasts, err := t.broadcastStorage.ClearBroadcasts(ctx) if err != nil { - return fmt.Errorf("%w: unable to clear broadcasts", err) + return fmt.Errorf("unable to clear broadcasts: %w", err) } log.Printf("cleared %d broadcasts\n", len(broadcasts)) @@ -481,7 +482,7 @@ func (t *ConstructionTester) PerformBroadcasts(ctx context.Context) error { color.Magenta("Rebroadcasting all transactions...") if err := t.broadcastStorage.BroadcastAll(ctx, false); err != nil { - return fmt.Errorf("%w: unable to broadcast all transactions", err) + return fmt.Errorf("unable to broadcast all transactions: %w", err) } return nil @@ -510,7 +511,7 @@ func (t *ConstructionTester) WatchEndConditions( for workflow, minOccurences := range endConditions { completed, err := t.jobStorage.Completed(ctx, workflow) if err != nil { - return fmt.Errorf("%w: unable to fetch completed %s", err, workflow) + return fmt.Errorf("unable to fetch completed %s: %w", workflow, err) } if len(completed) < minOccurences { @@ -579,7 +580,7 @@ func (t *ConstructionTester) HandleErr( t.config, t.counterStorage, t.jobStorage, - fmt.Errorf("%w: %v", customErrs.ErrConstructionCheckHalt, err.Error()), + fmt.Errorf("%v: %w", err.Error(), cliErrs.ErrConstructionCheckHalt), ) } diff --git a/pkg/tester/data.go b/pkg/tester/data.go index 12ed5a9d..e2584694 100644 --- a/pkg/tester/data.go +++ b/pkg/tester/data.go @@ -19,12 +19,13 @@ import ( "encoding/json" "errors" "fmt" - customErrs "github.com/coinbase/rosetta-cli/pkg/errors" "log" "math/big" "net/http" "time" + cliErrs "github.com/coinbase/rosetta-cli/pkg/errors" + "github.com/coinbase/rosetta-cli/configuration" "github.com/coinbase/rosetta-cli/pkg/logger" "github.com/coinbase/rosetta-cli/pkg/processor" @@ -63,18 +64,18 @@ const ( PeriodicLoggingFrequency = periodicLoggingSeconds * time.Second // EndAtTipCheckInterval is the frequency that EndAtTip condition - // is evaludated + // is evaluated EndAtTipCheckInterval = 10 * time.Second - + //MinTableSize unit is GB MinTableSize = int64(2) - + //MaxTableSize unit is GB MaxTableSize = int64(100) //MinTableSize unit is MB MinValueLogFileSize = int64(128) - + //MaxTableSize unit is MB MaxValueLogFileSize = int64(2048) ) @@ -127,7 +128,7 @@ func loadAccounts(filePath string) ([]*types.AccountCurrency, error) { accounts := []*types.AccountCurrency{} if err := utils.LoadAndParse(filePath, &accounts); err != nil { - return nil, fmt.Errorf("%w: unable to open account file", err) + return nil, fmt.Errorf("unable to load and parse %s: %w", filePath, err) } log.Printf( @@ -143,7 +144,7 @@ func loadAccounts(filePath string) ([]*types.AccountCurrency, error) { // CloseDatabase closes the database used by DataTester. func (t *DataTester) CloseDatabase(ctx context.Context) { if err := t.database.Close(ctx); err != nil { - log.Fatalf("%s: error closing database", err.Error()) + log.Fatalf("error closing database: %s", err.Error()) } } @@ -160,13 +161,13 @@ func InitializeData( ) (*DataTester, error) { dataPath, err := utils.CreateCommandPath(config.DataDirectory, dataCmdName, network) if err != nil { - return nil, fmt.Errorf("%s: cannot create command path", err.Error()) + return nil, fmt.Errorf("failed to create command path: %w", err) } opts := []database.BadgerOption{} dataPathBackup := dataPath - if config.AllInMemoryEnabled{ + if config.AllInMemoryEnabled { opts = append( opts, database.WithCustomSettings(database.AllInMemoryBadgerOptions(dataPath)), @@ -187,20 +188,20 @@ func InitializeData( } // If we enable all-in-memory or L0-in-memory mode, badger DB's TableSize and ValueLogFileSize will change - // according to users config. tableSize means the LSM table size, when the table more than the tableSize, - // will trigger a compact. + // according to users config. tableSize means the LSM table size, when the table more than the tableSize, + // will trigger a compact. // In default mode, we will not change the badger DB's TableSize and ValueLogFileSize for limiting memory usage if config.AllInMemoryEnabled || config.MemoryLimitDisabled { - if(config.TableSize != nil) { - if(*config.TableSize >= MinTableSize && *config.TableSize <= MaxTableSize) { + if config.TableSize != nil { + if *config.TableSize >= MinTableSize && *config.TableSize <= MaxTableSize { opts = append( opts, database.WithTableSize(*config.TableSize), ) } } - if(config.ValueLogFileSize != nil) { - if(*config.TableSize >= MinValueLogFileSize && *config.TableSize <= MinValueLogFileSize) { + if config.ValueLogFileSize != nil { + if *config.TableSize >= MinValueLogFileSize && *config.TableSize <= MinValueLogFileSize { opts = append( opts, database.WithValueLogFileSize(*config.TableSize), @@ -211,17 +212,17 @@ func InitializeData( localStore, err := database.NewBadgerDatabase(ctx, dataPathBackup, opts...) if err != nil { - return nil, fmt.Errorf("%s: unable to initialize database", err.Error()) + return nil, fmt.Errorf("unable to initialize database: %w", err) } exemptAccounts, err := loadAccounts(config.Data.ExemptAccounts) if err != nil { - return nil, fmt.Errorf("%s: unable to load exempt accounts", err.Error()) + return nil, fmt.Errorf("unable to load exempt accounts: %w", err) } interestingAccounts, err := loadAccounts(config.Data.InterestingAccounts) if err != nil { - return nil, fmt.Errorf("%s: unable to load interesting accounts", err.Error()) + return nil, fmt.Errorf("unable to load interesting accounts: %w", err) } counterStorage := modules.NewCounterStorage(localStore) @@ -238,7 +239,7 @@ func InitializeData( network, ) if err != nil { - return nil, fmt.Errorf("unable to initialize logger with error: %s", err.Error()) + return nil, fmt.Errorf("unable to initialize logger with error: %w", err) } var forceInactiveReconciliation bool @@ -262,12 +263,12 @@ func InitializeData( // Get all previously seen accounts seenAccounts, err := balanceStorage.GetAllAccountCurrency(ctx) if err != nil { - return nil, fmt.Errorf("%s: unable to get previously seen accounts", err.Error()) + return nil, fmt.Errorf("unable to get previously seen accounts: %w", err) } networkOptions, fetchErr := fetcher.NetworkOptionsRetry(ctx, network, nil) if fetchErr != nil { - log.Fatalf("%s: unable to get network options", fetchErr.Err.Error()) + log.Fatalf("unable to get network options: %s", fetchErr.Err.Error()) } if len(networkOptions.Allow.BalanceExemptions) > 0 && config.Data.InitialBalanceFetchDisabled { @@ -354,10 +355,10 @@ func InitializeData( genesisBlock, ) if err != nil { - return nil, fmt.Errorf("%s: unable to bootstrap balances", err.Error()) + return nil, fmt.Errorf("unable to bootstrap balances: %w", err) } case err != nil: - return nil, fmt.Errorf("%s: unable to get head block identifier", err.Error()) + return nil, fmt.Errorf("unable to get head block identifier: %w", err) default: log.Println("Skipping balance bootstrapping because already started syncing") } @@ -548,7 +549,7 @@ func (t *DataTester) syncedStatus(ctx context.Context) (bool, int64, error) { t.blockStorage, ) if err != nil { - return false, -1, err + return false, -1, fmt.Errorf("failed to check storage tip: %w", err) } var blockIndex int64 = -1 @@ -576,7 +577,7 @@ func (t *DataTester) EndAtTipLoop( atTip, blockIndex, err := t.syncedStatus(ctx) if err != nil { log.Printf( - "%s: unable to evaluate if syncer is at tip", + "unable to evaluate if syncer is at tip: %s", err.Error(), ) continue @@ -614,7 +615,7 @@ func (t *DataTester) EndReconciliationCoverage( // nolint:gocognit atTip, blockIndex, err := t.syncedStatus(ctx) if err != nil { log.Printf( - "%s: unable to evaluate syncer height or if at tip", + "unable to evaluate syncer height or if at tip: %s", err.Error(), ) continue @@ -675,7 +676,7 @@ func (t *DataTester) EndReconciliationCoverage( // nolint:gocognit allAccounts, err := t.balanceStorage.GetAllAccountCurrency(ctx) if err != nil { log.Printf( - "%s: unable to get account count", + "unable to get account count: %s", err.Error(), ) continue @@ -694,7 +695,7 @@ func (t *DataTester) EndReconciliationCoverage( // nolint:gocognit coverage, err := t.balanceStorage.ReconciliationCoverage(ctx, coverageIndex) if err != nil { log.Printf( - "%s: unable to get reconciliation coverage", + "unable to get reconciliation coverage: %s", err.Error(), ) continue @@ -776,22 +777,22 @@ func (t *DataTester) WatchEndConditions( func (t *DataTester) CompleteReconciliations(ctx context.Context) (int64, error) { activeReconciliations, err := t.counterStorage.Get(ctx, modules.ActiveReconciliationCounter) if err != nil { - return -1, fmt.Errorf("%w: cannot get active reconciliations counter", err) + return -1, fmt.Errorf("failed to get active reconciliations counter: %w", err) } exemptReconciliations, err := t.counterStorage.Get(ctx, modules.ExemptReconciliationCounter) if err != nil { - return -1, fmt.Errorf("%w: cannot get exempt reconciliations counter", err) + return -1, fmt.Errorf("failed to get exempt reconciliations counter: %w", err) } failedReconciliations, err := t.counterStorage.Get(ctx, modules.FailedReconciliationCounter) if err != nil { - return -1, fmt.Errorf("%w: cannot get failed reconciliations counter", err) + return -1, fmt.Errorf("failed to get failed reconciliations counter: %w", err) } skippedReconciliations, err := t.counterStorage.Get(ctx, modules.SkippedReconciliationsCounter) if err != nil { - return -1, fmt.Errorf("%w: cannot get skipped reconciliations counter", err) + return -1, fmt.Errorf("failed to get skipped reconciliations counter: %w", err) } return activeReconciliations.Int64() + @@ -811,7 +812,7 @@ func (t *DataTester) WaitForEmptyQueue( // and only exit when that many reconciliations have been performed. startingComplete, err := t.CompleteReconciliations(ctx) if err != nil { - return err + return fmt.Errorf("failed to complete reconciliations: %w", err) } startingRemaining := t.reconciler.QueueSize() @@ -832,12 +833,12 @@ func (t *DataTester) WaitForEmptyQueue( // We force cached counts to be written before // determining if we should exit. if err := t.reconcilerHandler.UpdateCounts(ctx); err != nil { - return err + return fmt.Errorf("failed to update count: %w", err) } nowComplete, err := t.CompleteReconciliations(ctx) if err != nil { - return err + return fmt.Errorf("failed to complete reconciliations: %w", err) } completed := nowComplete - startingComplete @@ -882,7 +883,7 @@ func (t *DataTester) DrainReconcilerQueue( err := g.Wait() if *t.signalReceived { - return errors.New("reconcilier queue drain halted") + return cliErrs.ErrReconcilerDrainHalt } if errors.Is(err, context.Canceled) { @@ -906,7 +907,7 @@ func (t *DataTester) HandleErr(err error, sigListeners *[]context.CancelFunc) er t.config, t.counterStorage, t.balanceStorage, - fmt.Errorf("%w: %v", customErrs.ErrDataCheckHalt, err.Error()), + fmt.Errorf("%v: %w", err.Error(), cliErrs.ErrDataCheckHalt), "", "", ) @@ -1015,7 +1016,7 @@ func (t *DataTester) FindMissingOps( t.reconcilerHandler.InactiveFailureBlock.Index, ) if err != nil { - color.Yellow("%s: could not find block with missing ops", err.Error()) + color.Yellow("could not find block with missing ops: %s", err.Error()) return results.ExitData( t.config, t.counterStorage, @@ -1057,13 +1058,13 @@ func (t *DataTester) recursiveOpSearch( // Always use a temporary directory to find missing ops tmpDir, err := utils.CreateTempDir() if err != nil { - return nil, fmt.Errorf("%w: unable to create temporary directory", err) + return nil, fmt.Errorf("unable to create temporary directory: %w", err) } defer utils.RemoveTempDir(tmpDir) localStore, err := database.NewBadgerDatabase(ctx, tmpDir) if err != nil { - return nil, fmt.Errorf("%w: unable to initialize database", err) + return nil, fmt.Errorf("unable to initialize database: %w", err) } counterStorage := modules.NewCounterStorage(localStore) @@ -1081,7 +1082,7 @@ func (t *DataTester) recursiveOpSearch( ) if err != nil { - return nil, fmt.Errorf("unable to initialize logger with error: %s", err.Error()) + return nil, fmt.Errorf("unable to initialize logger with error: %w", err) } t.forceInactiveReconciliation = types.Bool(false) @@ -1173,16 +1174,16 @@ func (t *DataTester) recursiveOpSearch( // Close database before starting another search, otherwise we will // have n databases open when we find the offending block. if storageErr := localStore.Close(ctx); storageErr != nil { - return nil, fmt.Errorf("%w: unable to close database", storageErr) + return nil, fmt.Errorf("unable to close database: %w", storageErr) } if *t.signalReceived { - return nil, errors.New("search for block with missing ops halted") + return nil, cliErrs.ErrMissingOps } if err == nil || errors.Is(err, context.Canceled) { if startIndex <= t.genesisBlock.Index { - return nil, errors.New("unable to find missing ops") + return nil, cliErrs.ErrUnableToFindMissingOps } newStart := startIndex - InactiveFailureLookbackWindow @@ -1193,7 +1194,7 @@ func (t *DataTester) recursiveOpSearch( newEnd := endIndex - InactiveFailureLookbackWindow if newEnd <= newStart { return nil, fmt.Errorf( - "Next window to check has start index %d <= end index %d", + "next window to check has start index %d <= end index %d", newStart, newEnd, ) @@ -1219,7 +1220,7 @@ func (t *DataTester) recursiveOpSearch( } if reconcilerHandler.ActiveFailureBlock == nil { - return nil, errors.New("unable to find missing ops") + return nil, cliErrs.ErrUnableToFindMissingOps } return reconcilerHandler.ActiveFailureBlock, nil diff --git a/pkg/tester/data_perf.go b/pkg/tester/data_perf.go index 0cdefc5c..0ce279ad 100644 --- a/pkg/tester/data_perf.go +++ b/pkg/tester/data_perf.go @@ -16,10 +16,10 @@ package tester import ( "context" - "errors" "time" "github.com/coinbase/rosetta-cli/configuration" + cliErrs "github.com/coinbase/rosetta-cli/pkg/errors" "github.com/coinbase/rosetta-cli/pkg/results" "github.com/coinbase/rosetta-sdk-go/fetcher" "github.com/coinbase/rosetta-sdk-go/types" @@ -56,7 +56,7 @@ func BmarkBlock(ctx context.Context, config *configuration.Configuration, fetche }() select { case <-ctx.Done(): - return errors.New("/block endpoint benchmarking timed out") + return cliErrs.ErrBlockBenchmarkTimeout case timeTaken := <-elapsed: rawStats.BlockEndpointTotalTime = timeTaken rawStats.BlockEndpointNumErrors = int64(total_errors) @@ -87,7 +87,7 @@ func BmarkAccountBalance(ctx context.Context, config *configuration.Configuratio }() select { case <-ctx.Done(): - return errors.New("/account/balance endpoint benchmarking timed out") + return cliErrs.ErrAccountBalanceBenchmarkTimeout case timeTaken := <-elapsed: rawStats.AccountBalanceEndpointTotalTime = timeTaken rawStats.AccountBalanceNumErrors = int64(total_errors) From 27df049009c1807f891dac4bc5c50781e450f8f2 Mon Sep 17 00:00:00 2001 From: Jingfu Wang Date: Mon, 19 Sep 2022 15:53:41 -0400 Subject: [PATCH 352/376] feat: decouple pruning configs Signed-off-by: Jingfu Wang --- configuration/types.go | 8 ++++++-- examples/configuration/default.json | 3 ++- pkg/processor/reconciler_helper.go | 2 +- pkg/tester/data.go | 2 +- 4 files changed, 10 insertions(+), 5 deletions(-) diff --git a/configuration/types.go b/configuration/types.go index 82afa7b3..e1ad000c 100644 --- a/configuration/types.go +++ b/configuration/types.go @@ -321,11 +321,15 @@ type DataConfiguration struct { // the results of a check:data run. ResultsOutputFile string `json:"results_output_file"` - // PruningDisabled is a bolean that indicates storage pruning should + // PruningBlockDisabled is a boolean that indicates storage pruning should // not be attempted. This should really only ever be set to true if you // wish to use `start_index` at a later point to restart from some // previously synced block. - PruningDisabled bool `json:"pruning_disabled"` + PruningBlockDisabled bool `json:"pruning_block_disabled"` + + // PruningBalanceDisabled is a boolean that indicates balance pruning + // should not be attempted. + PruningBalanceDisabled bool `json:"pruning_balance_disabled"` // PruningFrequency is the frequency (in seconds) that we attempt // to prune blocks. If not populated, we use the default value diff --git a/examples/configuration/default.json b/examples/configuration/default.json index c2ca45a7..355e334e 100644 --- a/examples/configuration/default.json +++ b/examples/configuration/default.json @@ -38,7 +38,8 @@ "coin_tracking_disabled": false, "status_port": 9090, "results_output_file": "", - "pruning_disabled": false, + "pruning_block_disabled": false, + "pruning_balance_disabled": false, "initial_balance_fetch_disabled": false }, "perf": null diff --git a/pkg/processor/reconciler_helper.go b/pkg/processor/reconciler_helper.go index dd50c760..a87fbc20 100644 --- a/pkg/processor/reconciler_helper.go +++ b/pkg/processor/reconciler_helper.go @@ -154,7 +154,7 @@ func (h *ReconcilerHelper) PruneBalances( currency *types.Currency, index int64, ) error { - if h.config.Data.PruningDisabled { + if h.config.Data.PruningBalanceDisabled { return nil } diff --git a/pkg/tester/data.go b/pkg/tester/data.go index e2584694..dcaec8df 100644 --- a/pkg/tester/data.go +++ b/pkg/tester/data.go @@ -443,7 +443,7 @@ func (t *DataTester) StartSyncing( func (t *DataTester) StartPruning( ctx context.Context, ) error { - if t.config.Data.PruningDisabled { + if t.config.Data.PruningBlockDisabled { return nil } From 6dd4f30c92e4610fd4d736c7a0bd138c337ddcdd Mon Sep 17 00:00:00 2001 From: Jingfu Wang Date: Mon, 19 Sep 2022 17:11:53 -0400 Subject: [PATCH 353/376] fix: error handling issues (#363) Signed-off-by: Jingfu Wang Signed-off-by: Jingfu Wang --- pkg/logger/logger.go | 6 +++++- pkg/processor/balance_storage_handler.go | 10 ++++++++-- 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/pkg/logger/logger.go b/pkg/logger/logger.go index 2bca8eda..c6f99ba7 100644 --- a/pkg/logger/logger.go +++ b/pkg/logger/logger.go @@ -286,7 +286,11 @@ func (l *Logger) RemoveBlockStream( ) fmt.Print(blockString) _, err = f.WriteString(blockString) - return fmt.Errorf("failed to write block string %s: %w", blockString, err) + if err != nil { + return fmt.Errorf("failed to write block string %s: %w", blockString, err) + } + + return nil } // TransactionStream writes the next processed block's transactions diff --git a/pkg/processor/balance_storage_handler.go b/pkg/processor/balance_storage_handler.go index 9dd251e8..d94f51d0 100644 --- a/pkg/processor/balance_storage_handler.go +++ b/pkg/processor/balance_storage_handler.go @@ -125,7 +125,10 @@ func (h *BalanceStorageHandler) AccountsReconciled( modules.ReconciledAccounts, big.NewInt(int64(count)), ) - return fmt.Errorf("failed to update the total accounts reconciled by count: %w", err) + if err != nil { + return fmt.Errorf("failed to update the total accounts reconciled by count: %w", err) + } + return nil } // AccountsSeen updates the total accounts seen by count. @@ -140,5 +143,8 @@ func (h *BalanceStorageHandler) AccountsSeen( modules.SeenAccounts, big.NewInt(int64(count)), ) - return fmt.Errorf("failed to update the total accounts seen by count: %w", err) + if err != nil { + return fmt.Errorf("failed to update the total accounts seen by count: %w", err) + } + return nil } From 0ff8d258e767d74408d0b5ec9426197ab4c49683 Mon Sep 17 00:00:00 2001 From: jingweicb <96205264+jingweicb@users.noreply.github.com> Date: Tue, 20 Sep 2022 11:47:30 -0700 Subject: [PATCH 354/376] expend the tablesize range (#347) * expend the tablesize range --- configuration/configuration.go | 2 +- configuration/types.go | 12 ++++++------ examples/configuration/default.json | 2 +- pkg/errors/errors.go | 2 +- pkg/tester/construction.go | 2 +- pkg/tester/data.go | 6 +++--- 6 files changed, 13 insertions(+), 13 deletions(-) diff --git a/configuration/configuration.go b/configuration/configuration.go index b2128a93..e3200e37 100644 --- a/configuration/configuration.go +++ b/configuration/configuration.go @@ -347,7 +347,7 @@ func assertConfiguration(ctx context.Context, config *Configuration) error { return fmt.Errorf("the number of serial block workers %d is invalid: %w", config.SerialBlockWorkers, cliErrs.ErrNegativeSerialBlockWorkers) } - if config.TableSize != nil && (*config.TableSize < 2 || *config.TableSize > 100) { + if config.TableSize != nil && (*config.TableSize < 1 || *config.TableSize > 100) { return fmt.Errorf("table size %d is invalid: %w", *config.TableSize, cliErrs.ErrTableSizeIsOutOfRange) } diff --git a/configuration/types.go b/configuration/types.go index e1ad000c..3de41acb 100644 --- a/configuration/types.go +++ b/configuration/types.go @@ -413,10 +413,10 @@ type Configuration struct { // to performance gains. CompressionDisabled bool `json:"compression_disabled"` - // MemoryLimitDisabled configures storage to increase memory + // L0InMemoryEnabled configures storage to increase memory // usage. Enabling this massively increases performance // but can use 10s of GBs of RAM, even with pruning enabled. - MemoryLimitDisabled bool `json:"memory_limit_disabled"` + L0InMemoryEnabled bool `json:"l0_in_memory_enabled"` // AllInMemoryDisabled configures storage to increase memory // usage. Enabling this massively increases performance @@ -424,12 +424,12 @@ type Configuration struct { AllInMemoryEnabled bool `json:"all_in_memory_enabled"` // TableSize unit is GB, enable users to define MaxTableSize - // when AllInMemoryEnabled == true or MemoryLimitDisabled== true, Cli will look up this config - // default value is 3, modification range is [2, 100] + // when AllInMemoryEnabled == true or L0InMemoryEnabled== true, Cli will look up this config + // default value is 2, modification range is [1, 100] TableSize *int64 `json:"table_size,omitempty"` - // TableSize unit is MB, enable users to define ValueLogFileSize - // when AllInMemoryEnabled == true or MemoryLimitDisabled== true, Cli will look up this config + // ValueLogFileSize unit is MB, enable users to define ValueLogFileSize + // when AllInMemoryEnabled == true or L0InMemoryEnabled== true, Cli will look up this config // default value is 512, modification range is [128, 2048] ValueLogFileSize *int64 `json:"value_log_file_size,omitempty"` diff --git a/examples/configuration/default.json b/examples/configuration/default.json index 355e334e..d2c0c4fc 100644 --- a/examples/configuration/default.json +++ b/examples/configuration/default.json @@ -14,7 +14,7 @@ "max_reorg_depth": 100, "log_configuration": false, "compression_disabled": false, - "memory_limit_disabled": false, + "l0_in_memory_enabled": false, "all_in_memory_enabled": false, "error_stack_trace_disabled": false, "coin_supported": false, diff --git a/pkg/errors/errors.go b/pkg/errors/errors.go index 5fa4293e..da824fa0 100644 --- a/pkg/errors/errors.go +++ b/pkg/errors/errors.go @@ -30,7 +30,7 @@ var ( ErrNegativeSeenBlockWorkers = errors.New("the number of seen block workers is negative") ErrNegativeSerialBlockWorkers = errors.New("the number of serial block workers is negative") ErrReconciliationOutOfRange = errors.New("reconciliation is out of range, it must be in the range [0, 1]") - ErrTableSizeIsOutOfRange = errors.New("table size is out of range, it must be in the range [2, 100]") + ErrTableSizeIsOutOfRange = errors.New("table size is out of range, it must be in the range [1, 100]") ErrValueLogFileSizeIsOutOfRange = errors.New("value log file size is out of range, it must be in the range [128, 2048]") ErrBalanceTrackingIsDisabledForReconciliation = errors.New("balance tracking cannot be disabled for reconciliation") ErrBalanceTrackingIsDisabledForReconciliationCoverageEndCondition = errors.New("balance tracking cannot be disabled for reconciliation coverage end condition") diff --git a/pkg/tester/construction.go b/pkg/tester/construction.go index e6d8d466..90e6701d 100644 --- a/pkg/tester/construction.go +++ b/pkg/tester/construction.go @@ -92,7 +92,7 @@ func InitializeConstruction( if config.CompressionDisabled { opts = append(opts, database.WithoutCompression()) } - if config.MemoryLimitDisabled { + if config.L0InMemoryEnabled { opts = append( opts, database.WithCustomSettings(database.PerformanceBadgerOptions(dataPath)), diff --git a/pkg/tester/data.go b/pkg/tester/data.go index dcaec8df..0d8f6033 100644 --- a/pkg/tester/data.go +++ b/pkg/tester/data.go @@ -68,7 +68,7 @@ const ( EndAtTipCheckInterval = 10 * time.Second //MinTableSize unit is GB - MinTableSize = int64(2) + MinTableSize = int64(1) //MaxTableSize unit is GB MaxTableSize = int64(100) @@ -179,7 +179,7 @@ func InitializeData( if config.CompressionDisabled { opts = append(opts, database.WithoutCompression()) } - if config.MemoryLimitDisabled { + if config.L0InMemoryEnabled { opts = append( opts, database.WithCustomSettings(database.PerformanceBadgerOptions(dataPath)), @@ -191,7 +191,7 @@ func InitializeData( // according to users config. tableSize means the LSM table size, when the table more than the tableSize, // will trigger a compact. // In default mode, we will not change the badger DB's TableSize and ValueLogFileSize for limiting memory usage - if config.AllInMemoryEnabled || config.MemoryLimitDisabled { + if config.AllInMemoryEnabled || config.L0InMemoryEnabled { if config.TableSize != nil { if *config.TableSize >= MinTableSize && *config.TableSize <= MaxTableSize { opts = append( From 085f95c85c99f607a82fb1814594d95dc9fefb55 Mon Sep 17 00:00:00 2001 From: Jingfu Wang Date: Tue, 20 Sep 2022 17:23:53 -0400 Subject: [PATCH 355/376] feat: version upgrade (#365) Signed-off-by: Jingfu Wang Signed-off-by: Jingfu Wang --- Dockerfile | 2 +- cmd/root.go | 2 +- go.mod | 2 +- go.sum | 4 ++-- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/Dockerfile b/Dockerfile index 928f3921..ae28c100 100644 --- a/Dockerfile +++ b/Dockerfile @@ -31,7 +31,7 @@ RUN mkdir -p "$GOPATH/src" "$GOPATH/bin" && chmod -R 777 "$GOPATH" WORKDIR /go/src -ARG VERSION=v0.9.0 +ARG VERSION=v0.10.0 RUN git clone https://github.com/coinbase/rosetta-cli.git && \ cd rosetta-cli && \ git fetch --all --tags && \ diff --git a/cmd/root.go b/cmd/root.go index 590eeddd..0b36a2fa 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -434,6 +434,6 @@ var versionCmd = &cobra.Command{ Use: "version", Short: "Print rosetta-cli version", Run: func(cmd *cobra.Command, args []string) { - fmt.Println("v0.9.0") + fmt.Println("v0.10.0") }, } diff --git a/go.mod b/go.mod index 1b0daef1..fdfc466e 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/coinbase/rosetta-cli go 1.16 require ( - github.com/coinbase/rosetta-sdk-go v0.8.0 + github.com/coinbase/rosetta-sdk-go v0.8.1 github.com/fatih/color v1.13.0 github.com/google/go-cmp v0.5.6 // indirect github.com/mattn/go-colorable v0.1.12 // indirect diff --git a/go.sum b/go.sum index b8dc4796..2d23f0e9 100644 --- a/go.sum +++ b/go.sum @@ -98,8 +98,8 @@ github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDk github.com/cloudflare/cloudflare-go v0.14.0/go.mod h1:EnwdgGMaFOruiPZRFSgn+TsQ3hQ7C/YWzIGLeu5c304= github.com/coinbase/kryptology v1.8.0 h1:Aoq4gdTsJhSU3lNWsD5BWmFSz2pE0GlmrljaOxepdYY= github.com/coinbase/kryptology v1.8.0/go.mod h1:RYXOAPdzOGUe3qlSFkMGn58i3xUA8hmxYHksuq+8ciI= -github.com/coinbase/rosetta-sdk-go v0.8.0 h1:FXootIYNrQyVbpGfWfmEOBzEGKku8Wuyql+iyIy3L+g= -github.com/coinbase/rosetta-sdk-go v0.8.0/go.mod h1:tXPR6AIW9ogsH4tYIaFOKOgfJNanCvcyl7JKLd4DToc= +github.com/coinbase/rosetta-sdk-go v0.8.1 h1:WE+Temc8iz7Ra7sCpV9ymBJx78vItqFJ2xcSiPet1Pc= +github.com/coinbase/rosetta-sdk-go v0.8.1/go.mod h1:tXPR6AIW9ogsH4tYIaFOKOgfJNanCvcyl7JKLd4DToc= github.com/consensys/bavard v0.1.8-0.20210406032232-f3452dc9b572/go.mod h1:Bpd0/3mZuaj6Sj+PqrmIquiOKy397AKGThQPaGzNXAQ= github.com/consensys/bavard v0.1.8-0.20210915155054-088da2f7f54a/go.mod h1:9ItSMtA/dXMAiL7BG6bqW2m3NdSEObYWoH223nGHukI= github.com/consensys/gnark-crypto v0.4.1-0.20210426202927-39ac3d4b3f1f/go.mod h1:815PAHg3wvysy0SyIqanF8gZ0Y1wjk/hrDHD/iT88+Q= From 11cc4aba05156774cc44b9d2e4370c86847e1064 Mon Sep 17 00:00:00 2001 From: Madhur Shrimal Date: Wed, 7 Dec 2022 19:08:32 -0500 Subject: [PATCH 356/376] [fix] bootstrap balances should be done only for genesis block testing (#375) * bootstrap balances should be done only for genesis block testing * add header timeout --- pkg/tester/data.go | 7 ++++++- pkg/tester/general.go | 8 ++++++-- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/pkg/tester/data.go b/pkg/tester/data.go index 0d8f6033..d3d9f568 100644 --- a/pkg/tester/data.go +++ b/pkg/tester/data.go @@ -345,7 +345,12 @@ func InitializeData( // // We need to do this after instantiating the balance storage handler // because it is invoked within BootstrapBalances. - if len(config.Data.BootstrapBalances) > 0 { + // + // We only need to bootstrap balances when we run this test from + // genesis block. If it is not genesis block, we use the balances from + // previous block + if (config.Data.StartIndex == nil || *config.Data.StartIndex == genesisBlock.Index) && + len(config.Data.BootstrapBalances) > 0 { _, err := blockStorage.GetHeadBlockIdentifier(ctx) switch { case err == storageErrs.ErrHeadBlockNotFound: diff --git a/pkg/tester/general.go b/pkg/tester/general.go index 776070ce..0edc1a3f 100644 --- a/pkg/tester/general.go +++ b/pkg/tester/general.go @@ -28,6 +28,9 @@ const ( // MemoryLoggingFrequency is the frequency that memory // usage stats are logged to the terminal. MemoryLoggingFrequency = 10 * time.Second + + // ReadHeaderTimeout is the header timeout for server + ReadHeaderTimeout = 5 * time.Second ) // LogMemoryLoop runs a loop that logs memory usage. @@ -57,8 +60,9 @@ func StartServer( port uint, ) error { server := &http.Server{ - Addr: fmt.Sprintf(":%d", port), - Handler: handler, + Addr: fmt.Sprintf(":%d", port), + Handler: handler, + ReadHeaderTimeout: ReadHeaderTimeout, } go func() { From e384ab3869f6446a63d4fca5a5cade665744b708 Mon Sep 17 00:00:00 2001 From: jingwei zhuang Date: Thu, 8 Dec 2022 00:20:46 -0800 Subject: [PATCH 357/376] init requestUUID --- cmd/check_construction.go | 3 ++ cmd/check_data.go | 3 ++ cmd/root.go | 20 +++++++++++ configuration/types.go | 3 ++ pkg/logger/logger.go | 69 +++++++++++++++++++++++++++++++++----- pkg/tester/construction.go | 1 + pkg/tester/data.go | 5 +++ 7 files changed, 96 insertions(+), 8 deletions(-) diff --git a/cmd/check_construction.go b/cmd/check_construction.go index 497497d7..c05c6745 100644 --- a/cmd/check_construction.go +++ b/cmd/check_construction.go @@ -20,6 +20,7 @@ import ( "time" cliErrs "github.com/coinbase/rosetta-cli/pkg/errors" + "github.com/coinbase/rosetta-cli/pkg/logger" "github.com/coinbase/rosetta-cli/pkg/results" "github.com/coinbase/rosetta-cli/pkg/tester" @@ -147,6 +148,8 @@ func runCheckConstructionCmd(_ *cobra.Command, _ []string) error { } g, ctx := errgroup.WithContext(ctx) + ctx = logger.AddRequestUUIDToContext(ctx, Config.RequestUUID) + g.Go(func() error { return constructionTester.StartPeriodicLogger(ctx) }) diff --git a/cmd/check_data.go b/cmd/check_data.go index 962f3647..81637654 100644 --- a/cmd/check_data.go +++ b/cmd/check_data.go @@ -19,6 +19,7 @@ import ( "fmt" "time" + "github.com/coinbase/rosetta-cli/pkg/logger" "github.com/coinbase/rosetta-cli/pkg/results" "github.com/coinbase/rosetta-cli/pkg/tester" "github.com/coinbase/rosetta-sdk-go/fetcher" @@ -154,6 +155,8 @@ func runCheckDataCmd(_ *cobra.Command, _ []string) error { defer dataTester.CloseDatabase(ctx) g, ctx := errgroup.WithContext(ctx) + ctx = logger.AddRequestUUIDToContext(ctx, Config.RequestUUID) + g.Go(func() error { return dataTester.StartPeriodicLogger(ctx) }) diff --git a/cmd/root.go b/cmd/root.go index 0b36a2fa..6c9c3fc7 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -57,6 +57,7 @@ var ( dataDirectory string inMemoryMode bool tableSize int64 + requestUUID string // Config is the populated *configuration.Configuration from // the configurationFile. If none is provided, this is set @@ -280,6 +281,13 @@ default values.`, "In-memory-mode configures badger DB inMeomry option. Only when in-memory-mode=true, this will override the all_in_memory_enabled", ) + checkDataCmd.Flags().StringVar( + &requestUUID, + "requestUUID", + "", + "requestUUID configures the requestUUID in logs, which aims to enable search logs by requestUUID", + ) + rootCmd.AddCommand(checkDataCmd) checkConstructionCmd.Flags().StringVar( &asserterConfigurationFile, @@ -308,6 +316,14 @@ default values.`, "", "Result-file configures the location of validation result. This will override the results_output_file from configuration file", ) + + checkConstructionCmd.Flags().StringVar( + &requestUUID, + "requestUUID", + "", + "requestUUID configures the requestUUID in logs, which aims to enable search logs by requestUUID", + ) + rootCmd.AddCommand(checkConstructionCmd) // View Commands @@ -399,6 +415,10 @@ func initConfig() { } else if tableSize != -1 { log.Fatalf("table-size %d is not in the range [2, 100], please check your input", tableSize) } + + if len(requestUUID) != 0 { + Config.RequestUUID = requestUUID + } } func ensureDataDirectoryExists() { diff --git a/configuration/types.go b/configuration/types.go index 3de41acb..b7b5d7bc 100644 --- a/configuration/types.go +++ b/configuration/types.go @@ -367,6 +367,9 @@ type Configuration struct { // binary is being executed. DataDirectory string `json:"data_directory"` + // Make search log easier when validation pool is working on different request + RequestUUID string `json:"requestUUID,omitempty"` + // HTTPTimeout is the timeout for a HTTP request in seconds. HTTPTimeout uint64 `json:"http_timeout"` diff --git a/pkg/logger/logger.go b/pkg/logger/logger.go index c6f99ba7..0effac4b 100644 --- a/pkg/logger/logger.go +++ b/pkg/logger/logger.go @@ -38,7 +38,11 @@ var _ statefulsyncer.Logger = (*Logger)(nil) type CheckType string +type contextKey int + const ( + RequestUUID contextKey = iota + // blockStreamFile contains the stream of processed // blocks and whether they were added or removed. blockStreamFile = "blocks.txt" @@ -78,6 +82,7 @@ type Logger struct { logTransactions bool logBalanceChanges bool logReconciliation bool + logRequestUUID string lastStatsMessage string lastProgressMessage string @@ -94,9 +99,10 @@ func NewLogger( logReconciliation bool, checkType CheckType, network *types.NetworkIdentifier, + logRequestUUID string, fields ...zap.Field, ) (*Logger, error) { - zapLogger, err := buildZapLogger(checkType, network, fields...) + zapLogger, err := buildZapLogger(checkType, network, logRequestUUID, fields...) if err != nil { return nil, fmt.Errorf("failed to build zap logger: %w", err) } @@ -106,6 +112,7 @@ func NewLogger( logTransactions: logTransactions, logBalanceChanges: logBalanceChanges, logReconciliation: logReconciliation, + logRequestUUID: logRequestUUID, zapLogger: zapLogger, }, nil } @@ -113,6 +120,7 @@ func NewLogger( func buildZapLogger( checkType CheckType, network *types.NetworkIdentifier, + requestUUID string, fields ...zap.Field, ) (*zap.Logger, error) { config := zap.NewDevelopmentConfig() @@ -121,6 +129,7 @@ func buildZapLogger( baseSlice := []zap.Field{ zap.String("blockchain", network.Blockchain), zap.String("network", network.Network), + zap.String("requestUUID", requestUUID), zap.String("check_type", string(checkType)), } mergedSlice := append(baseSlice, fields...) @@ -151,6 +160,8 @@ func (l *Logger) LogDataStatus(ctx context.Context, status *results.CheckDataSta status.Stats.ReconciliationCoverage*utils.OneHundred, ) + statsMessage = AddRequestUUID(statsMessage, l.logRequestUUID) + // Don't print out the same stats message twice. if statsMessage == l.lastStatsMessage { return @@ -175,6 +186,8 @@ func (l *Logger) LogDataStatus(ctx context.Context, status *results.CheckDataSta status.Progress.ReconcilerLastIndex, ) + progressMessage = AddRequestUUID(progressMessage, l.logRequestUUID) + // Don't print out the same progress message twice. if progressMessage == l.lastProgressMessage { return @@ -202,6 +215,8 @@ func (l *Logger) LogConstructionStatus( return } + statsMessage = AddRequestUUID(statsMessage, l.logRequestUUID) + l.lastStatsMessage = statsMessage color.Cyan(statsMessage) } @@ -216,7 +231,7 @@ func LogMemoryStats(ctx context.Context) { memUsage.System, memUsage.GarbageCollections, ) - + statsMessage = AddRequestUUIDFromContext(ctx, statsMessage) color.Cyan(statsMessage) } @@ -242,12 +257,13 @@ func (l *Logger) AddBlockStream( defer closeFile(f) blockString := fmt.Sprintf( - "%s Block %d:%s with Parent Block %d:%s\n", + "%s Block %d:%s with Parent Block %d:%s , RequestUUID: %s\n", addEvent, block.BlockIdentifier.Index, block.BlockIdentifier.Hash, block.ParentBlockIdentifier.Index, block.ParentBlockIdentifier.Hash, + l.logRequestUUID, ) fmt.Print(blockString) if _, err := f.WriteString(blockString); err != nil { @@ -279,10 +295,11 @@ func (l *Logger) RemoveBlockStream( defer closeFile(f) blockString := fmt.Sprintf( - "%s Block %d:%s\n", + "%s Block %d:%s , RequestUUID: %s\n", removeEvent, block.Index, block.Hash, + l.logRequestUUID, ) fmt.Print(blockString) _, err = f.WriteString(blockString) @@ -316,10 +333,11 @@ func (l *Logger) TransactionStream( for _, tx := range block.Transactions { transactionString := fmt.Sprintf( - "Transaction %s at Block %d:%s\n", + "Transaction %s at Block %d:%s , RequestUUID: %s\n", tx.TransactionIdentifier.Hash, block.BlockIdentifier.Index, block.BlockIdentifier.Hash, + l.logRequestUUID, ) fmt.Print(transactionString) _, err = f.WriteString(transactionString) @@ -394,7 +412,7 @@ func (l *Logger) BalanceStream( balanceChange.Block.Index, balanceChange.Block.Hash, ) - + balanceLog = AddRequestUUID(balanceLog, l.logRequestUUID) if _, err := f.WriteString(fmt.Sprintf("%s\n", balanceLog)); err != nil { return fmt.Errorf("failed to write balance log %s: %w", balanceLog, err) } @@ -435,13 +453,14 @@ func (l *Logger) ReconcileSuccessStream( ) reconciliationSuccessString := fmt.Sprintf( - "Type:%s Account: %s Currency: %s Balance: %s Block: %d:%s\n", + "Type:%s Account: %s Currency: %s Balance: %s Block: %d:%s , RequestUUID: %s\n", reconciliationType, types.AccountString(account), types.CurrencyString(currency), balance, block.Index, block.Hash, + l.logRequestUUID, ) _, err = f.WriteString(reconciliationSuccessString) if err != nil { @@ -500,7 +519,7 @@ func (l *Logger) ReconcileFailureStream( defer closeFile(f) reconciliationFailureString := fmt.Sprintf( - "Type:%s Account: %s Currency: %s Block: %s:%d computed: %s live: %s\n", + "Type:%s Account: %s Currency: %s Block: %s:%d computed: %s live: %s , RequestUUID: %s\n", reconciliationType, types.AccountString(account), types.CurrencyString(currency), @@ -508,6 +527,7 @@ func (l *Logger) ReconcileFailureStream( block.Index, computedBalance, liveBalance, + l.logRequestUUID, ) _, err = f.WriteString(reconciliationFailureString) if err != nil { @@ -565,3 +585,36 @@ func LogTransactionCreated( transactionIdentifier.Hash, ) } + +// Add requestUUID to the tip +func AddRequestUUIDFromContext(ctx context.Context, msg string) string { + requestUUID := requestUUIDFromContext(ctx) + if requestUUID != "" { + msg = fmt.Sprintf("%s, RequestUUID: %s\n", msg, requestUUID) + } + return msg +} + + +// Add requestUUID to the tip +func AddRequestUUID(msg string, requestUUID string) string { + if requestUUID != "" { + msg = fmt.Sprintf("%s, RequestUUID: %s\n", msg, requestUUID) + } + return msg +} + +// AddRequestUUIDToContext will add a requestUUIDto the context, and return the new context +func AddRequestUUIDToContext(ctx context.Context, uuid string) context.Context { + return context.WithValue(ctx, RequestUUID, uuid) +} + +// requestUUIDFromContext is used to extract a request UUID from a context +func requestUUIDFromContext(ctx context.Context) string { + switch v := ctx.Value(RequestUUID).(type) { + case string: + return v + default: + return "" + } +} \ No newline at end of file diff --git a/pkg/tester/construction.go b/pkg/tester/construction.go index 90e6701d..ba8a806f 100644 --- a/pkg/tester/construction.go +++ b/pkg/tester/construction.go @@ -123,6 +123,7 @@ func InitializeConstruction( false, logger.Construction, network, + config.RequestUUID, ) if err != nil { return nil, fmt.Errorf("unable to initialize logger with error: %w", err) diff --git a/pkg/tester/data.go b/pkg/tester/data.go index 0d8f6033..10fd6658 100644 --- a/pkg/tester/data.go +++ b/pkg/tester/data.go @@ -78,6 +78,9 @@ const ( //MaxTableSize unit is MB MaxValueLogFileSize = int64(2048) + + // empty requestUUID + EmptyRequestUUID = "" ) var _ http.Handler = (*DataTester)(nil) @@ -237,6 +240,7 @@ func InitializeData( config.Data.LogReconciliations, logger.Data, network, + config.RequestUUID, ) if err != nil { return nil, fmt.Errorf("unable to initialize logger with error: %w", err) @@ -1079,6 +1083,7 @@ func (t *DataTester) recursiveOpSearch( false, logger.Data, t.network, + EmptyRequestUUID, ) if err != nil { From 5a2e95153bf82e8f98876fac433cedf6aa5e84f0 Mon Sep 17 00:00:00 2001 From: jingwei zhuang Date: Thu, 8 Dec 2022 00:26:39 -0800 Subject: [PATCH 358/376] update --- pkg/logger/logger.go | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/pkg/logger/logger.go b/pkg/logger/logger.go index 0effac4b..7a2ad64b 100644 --- a/pkg/logger/logger.go +++ b/pkg/logger/logger.go @@ -112,7 +112,7 @@ func NewLogger( logTransactions: logTransactions, logBalanceChanges: logBalanceChanges, logReconciliation: logReconciliation, - logRequestUUID: logRequestUUID, + logRequestUUID: logRequestUUID, zapLogger: zapLogger, }, nil } @@ -595,7 +595,6 @@ func AddRequestUUIDFromContext(ctx context.Context, msg string) string { return msg } - // Add requestUUID to the tip func AddRequestUUID(msg string, requestUUID string) string { if requestUUID != "" { @@ -617,4 +616,4 @@ func requestUUIDFromContext(ctx context.Context) string { default: return "" } -} \ No newline at end of file +} From e12a63c5a9170a87fd5f252f61ec370ff9354116 Mon Sep 17 00:00:00 2001 From: jingwei zhuang Date: Thu, 8 Dec 2022 14:02:27 -0800 Subject: [PATCH 359/376] updates --- pkg/logger/logger.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/logger/logger.go b/pkg/logger/logger.go index 7a2ad64b..3dc4390a 100644 --- a/pkg/logger/logger.go +++ b/pkg/logger/logger.go @@ -123,7 +123,7 @@ func buildZapLogger( requestUUID string, fields ...zap.Field, ) (*zap.Logger, error) { - config := zap.NewDevelopmentConfig() + config := zap.NewProductionConfig() config.EncoderConfig.EncodeLevel = zapcore.CapitalColorLevelEncoder baseSlice := []zap.Field{ From 8fdd6ed31924bc33702300f4707bace70c25f6bd Mon Sep 17 00:00:00 2001 From: jingwei zhuang Date: Thu, 8 Dec 2022 14:24:33 -0800 Subject: [PATCH 360/376] updates --- pkg/logger/logger.go | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/pkg/logger/logger.go b/pkg/logger/logger.go index 3dc4390a..6eda3f9c 100644 --- a/pkg/logger/logger.go +++ b/pkg/logger/logger.go @@ -257,7 +257,7 @@ func (l *Logger) AddBlockStream( defer closeFile(f) blockString := fmt.Sprintf( - "%s Block %d:%s with Parent Block %d:%s , RequestUUID: %s\n", + "%s Block %d:%s with Parent Block %d:%s ,RequestUUID: %s\n", addEvent, block.BlockIdentifier.Index, block.BlockIdentifier.Hash, @@ -295,7 +295,7 @@ func (l *Logger) RemoveBlockStream( defer closeFile(f) blockString := fmt.Sprintf( - "%s Block %d:%s , RequestUUID: %s\n", + "%s Block %d:%s ,RequestUUID: %s\n", removeEvent, block.Index, block.Hash, @@ -333,7 +333,7 @@ func (l *Logger) TransactionStream( for _, tx := range block.Transactions { transactionString := fmt.Sprintf( - "Transaction %s at Block %d:%s , RequestUUID: %s\n", + "Transaction %s at Block %d:%s ,RequestUUID: %s\n", tx.TransactionIdentifier.Hash, block.BlockIdentifier.Index, block.BlockIdentifier.Hash, @@ -453,7 +453,7 @@ func (l *Logger) ReconcileSuccessStream( ) reconciliationSuccessString := fmt.Sprintf( - "Type:%s Account: %s Currency: %s Balance: %s Block: %d:%s , RequestUUID: %s\n", + "Type:%s Account: %s Currency: %s Balance: %s Block: %d:%s ,RequestUUID: %s\n", reconciliationType, types.AccountString(account), types.CurrencyString(currency), @@ -519,7 +519,7 @@ func (l *Logger) ReconcileFailureStream( defer closeFile(f) reconciliationFailureString := fmt.Sprintf( - "Type:%s Account: %s Currency: %s Block: %s:%d computed: %s live: %s , RequestUUID: %s\n", + "Type:%s Account: %s Currency: %s Block: %s:%d computed: %s live: %s ,RequestUUID: %s\n", reconciliationType, types.AccountString(account), types.CurrencyString(currency), @@ -590,7 +590,7 @@ func LogTransactionCreated( func AddRequestUUIDFromContext(ctx context.Context, msg string) string { requestUUID := requestUUIDFromContext(ctx) if requestUUID != "" { - msg = fmt.Sprintf("%s, RequestUUID: %s\n", msg, requestUUID) + msg = fmt.Sprintf("%s,RequestUUID: %s\n", msg, requestUUID) } return msg } @@ -598,7 +598,7 @@ func AddRequestUUIDFromContext(ctx context.Context, msg string) string { // Add requestUUID to the tip func AddRequestUUID(msg string, requestUUID string) string { if requestUUID != "" { - msg = fmt.Sprintf("%s, RequestUUID: %s\n", msg, requestUUID) + msg = fmt.Sprintf("%s,RequestUUID: %s\n", msg, requestUUID) } return msg } From e75e1c4130712e23405c2a902ae44243ebd87a90 Mon Sep 17 00:00:00 2001 From: jingwei zhuang Date: Thu, 8 Dec 2022 14:28:57 -0800 Subject: [PATCH 361/376] updates --- pkg/logger/logger.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/pkg/logger/logger.go b/pkg/logger/logger.go index 6eda3f9c..72e79047 100644 --- a/pkg/logger/logger.go +++ b/pkg/logger/logger.go @@ -257,7 +257,7 @@ func (l *Logger) AddBlockStream( defer closeFile(f) blockString := fmt.Sprintf( - "%s Block %d:%s with Parent Block %d:%s ,RequestUUID: %s\n", + "%s Block %d:%s with Parent Block %d:%s, RequestUUID: %s\n", addEvent, block.BlockIdentifier.Index, block.BlockIdentifier.Hash, @@ -295,7 +295,7 @@ func (l *Logger) RemoveBlockStream( defer closeFile(f) blockString := fmt.Sprintf( - "%s Block %d:%s ,RequestUUID: %s\n", + "%s Block %d:%s, RequestUUID: %s\n", removeEvent, block.Index, block.Hash, @@ -333,7 +333,7 @@ func (l *Logger) TransactionStream( for _, tx := range block.Transactions { transactionString := fmt.Sprintf( - "Transaction %s at Block %d:%s ,RequestUUID: %s\n", + "Transaction %s at Block %d:%s, RequestUUID: %s\n", tx.TransactionIdentifier.Hash, block.BlockIdentifier.Index, block.BlockIdentifier.Hash, @@ -453,7 +453,7 @@ func (l *Logger) ReconcileSuccessStream( ) reconciliationSuccessString := fmt.Sprintf( - "Type:%s Account: %s Currency: %s Balance: %s Block: %d:%s ,RequestUUID: %s\n", + "Type:%s Account: %s Currency: %s Balance: %s Block: %d:%s, RequestUUID: %s\n", reconciliationType, types.AccountString(account), types.CurrencyString(currency), @@ -519,7 +519,7 @@ func (l *Logger) ReconcileFailureStream( defer closeFile(f) reconciliationFailureString := fmt.Sprintf( - "Type:%s Account: %s Currency: %s Block: %s:%d computed: %s live: %s ,RequestUUID: %s\n", + "Type:%s Account: %s Currency: %s Block: %s:%d computed: %s live: %s, RequestUUID: %s\n", reconciliationType, types.AccountString(account), types.CurrencyString(currency), From 585f9dd74ae60766e41c91a6ee682557c20aa34b Mon Sep 17 00:00:00 2001 From: jingweicb <96205264+jingweicb@users.noreply.github.com> Date: Thu, 8 Dec 2022 17:25:22 -0800 Subject: [PATCH 362/376] change fmt to log (#377) --- pkg/logger/logger.go | 100 ++++++++++++++++++++++++++++--------------- 1 file changed, 66 insertions(+), 34 deletions(-) diff --git a/pkg/logger/logger.go b/pkg/logger/logger.go index 72e79047..cd2aebe4 100644 --- a/pkg/logger/logger.go +++ b/pkg/logger/logger.go @@ -124,7 +124,7 @@ func buildZapLogger( fields ...zap.Field, ) (*zap.Logger, error) { config := zap.NewProductionConfig() - config.EncoderConfig.EncodeLevel = zapcore.CapitalColorLevelEncoder + config.EncoderConfig.EncodeLevel = zapcore.CapitalLevelEncoder baseSlice := []zap.Field{ zap.String("blockchain", network.Blockchain), @@ -168,7 +168,7 @@ func (l *Logger) LogDataStatus(ctx context.Context, status *results.CheckDataSta } l.lastStatsMessage = statsMessage - color.Cyan(statsMessage) + l.zapLogger.Info(statsMessage) // If Progress is nil, it means we're already done. if status.Progress == nil { @@ -194,7 +194,7 @@ func (l *Logger) LogDataStatus(ctx context.Context, status *results.CheckDataSta } l.lastProgressMessage = progressMessage - color.Cyan(progressMessage) + l.zapLogger.Info(progressMessage) } // LogConstructionStatus logs results.CheckConstructionStatus. @@ -218,7 +218,7 @@ func (l *Logger) LogConstructionStatus( statsMessage = AddRequestUUID(statsMessage, l.logRequestUUID) l.lastStatsMessage = statsMessage - color.Cyan(statsMessage) + l.zapLogger.Info(statsMessage) } // LogMemoryStats logs memory usage information. @@ -251,21 +251,23 @@ func (l *Logger) AddBlockStream( os.FileMode(utils.DefaultFilePermissions), ) if err != nil { - return fmt.Errorf("failed to open file %s: %w", path.Join(l.logDir, blockStreamFile), err) + err = fmt.Errorf("failed to open file %s: %w", path.Join(l.logDir, blockStreamFile), err) + l.zapLogger.Error(err.Error()) + return err } defer closeFile(f) blockString := fmt.Sprintf( - "%s Block %d:%s with Parent Block %d:%s, RequestUUID: %s\n", + "%s Block %d:%s with Parent Block %d:%s", addEvent, block.BlockIdentifier.Index, block.BlockIdentifier.Hash, block.ParentBlockIdentifier.Index, block.ParentBlockIdentifier.Hash, - l.logRequestUUID, ) - fmt.Print(blockString) + blockString = AddRequestUUID(blockString, l.logRequestUUID) + l.zapLogger.Info(blockString) if _, err := f.WriteString(blockString); err != nil { return fmt.Errorf("failed to write block string %s: %w", blockString, err) } @@ -289,22 +291,26 @@ func (l *Logger) RemoveBlockStream( os.FileMode(utils.DefaultFilePermissions), ) if err != nil { - return fmt.Errorf("failed to open file %s: %w", path.Join(l.logDir, blockStreamFile), err) + err = fmt.Errorf("failed to open file %s: %w", path.Join(l.logDir, blockStreamFile), err) + l.zapLogger.Error(err.Error()) + return err } defer closeFile(f) blockString := fmt.Sprintf( - "%s Block %d:%s, RequestUUID: %s\n", + "%s Block %d:%s", removeEvent, block.Index, block.Hash, - l.logRequestUUID, ) - fmt.Print(blockString) + blockString = AddRequestUUID(blockString, l.logRequestUUID) + l.zapLogger.Info(blockString) _, err = f.WriteString(blockString) if err != nil { - return fmt.Errorf("failed to write block string %s: %w", blockString, err) + err = fmt.Errorf("failed to write block string %s: %w", blockString, err) + l.zapLogger.Error(err.Error()) + return err } return nil @@ -326,23 +332,27 @@ func (l *Logger) TransactionStream( os.FileMode(utils.DefaultFilePermissions), ) if err != nil { - return fmt.Errorf("failed to open file %s: %w", path.Join(l.logDir, transactionStreamFile), err) + err = fmt.Errorf("failed to open file %s: %w", path.Join(l.logDir, transactionStreamFile), err) + l.zapLogger.Error(err.Error()) + return err } defer closeFile(f) for _, tx := range block.Transactions { transactionString := fmt.Sprintf( - "Transaction %s at Block %d:%s, RequestUUID: %s\n", + "Transaction %s at Block %d:%s", tx.TransactionIdentifier.Hash, block.BlockIdentifier.Index, block.BlockIdentifier.Hash, - l.logRequestUUID, ) - fmt.Print(transactionString) + transactionString = AddRequestUUID(transactionString, l.logRequestUUID) + l.zapLogger.Info(transactionString) _, err = f.WriteString(transactionString) if err != nil { - return fmt.Errorf("failed to write transaction string %s: %w", transactionString, err) + err = fmt.Errorf("failed to write transaction string %s: %w", transactionString, err) + l.zapLogger.Error(err.Error()) + return err } for _, op := range tx.Operations { @@ -363,7 +373,7 @@ func (l *Logger) TransactionStream( } transactionOperationString := fmt.Sprintf( - "TxOp %d(%d) %s %s %s %s %s\n", + "TxOp %d(%d) %s %s %s %s %s", op.OperationIdentifier.Index, networkIndex, op.Type, @@ -372,9 +382,13 @@ func (l *Logger) TransactionStream( symbol, *op.Status, ) + transactionOperationString = AddRequestUUID(transactionOperationString, l.logRequestUUID) + l.zapLogger.Info(transactionOperationString) _, err = f.WriteString(transactionOperationString) if err != nil { - return fmt.Errorf("failed to write transaction operation string %s: %w", transactionOperationString, err) + err = fmt.Errorf("failed to write transaction operation string %s: %w", transactionOperationString, err) + l.zapLogger.Error(err.Error()) + return err } } } @@ -398,7 +412,9 @@ func (l *Logger) BalanceStream( os.FileMode(utils.DefaultFilePermissions), ) if err != nil { - return fmt.Errorf("failed to open file %s: %w", path.Join(l.logDir, balanceStreamFile), err) + err = fmt.Errorf("failed to open file %s: %w", path.Join(l.logDir, balanceStreamFile), err) + l.zapLogger.Error(err.Error()) + return err } defer closeFile(f) @@ -413,8 +429,11 @@ func (l *Logger) BalanceStream( balanceChange.Block.Hash, ) balanceLog = AddRequestUUID(balanceLog, l.logRequestUUID) + l.zapLogger.Info(balanceLog) if _, err := f.WriteString(fmt.Sprintf("%s\n", balanceLog)); err != nil { - return fmt.Errorf("failed to write balance log %s: %w", balanceLog, err) + err = fmt.Errorf("failed to write balance log %s: %w", balanceLog, err) + l.zapLogger.Error(err.Error()) + return err } } return nil @@ -440,31 +459,39 @@ func (l *Logger) ReconcileSuccessStream( os.FileMode(utils.DefaultFilePermissions), ) if err != nil { - return fmt.Errorf("failed to open file %s: %w", path.Join(l.logDir, reconcileSuccessStreamFile), err) + err = fmt.Errorf("failed to open file %s: %w", path.Join(l.logDir, reconcileSuccessStreamFile), err) + l.zapLogger.Error(err.Error()) + return err } defer closeFile(f) - log.Printf( - "%s Reconciled %s at %d\n", + reconciledLog := fmt.Sprintf( + "%s Reconciled %s at %d", reconciliationType, types.AccountString(account), block.Index, ) + reconciledLog = AddRequestUUID(reconciledLog, l.logRequestUUID) + l.zapLogger.Info(reconciledLog) reconciliationSuccessString := fmt.Sprintf( - "Type:%s Account: %s Currency: %s Balance: %s Block: %d:%s, RequestUUID: %s\n", + "Type:%s Account: %s Currency: %s Balance: %s Block: %d:%s", reconciliationType, types.AccountString(account), types.CurrencyString(currency), balance, block.Index, block.Hash, - l.logRequestUUID, ) + reconciliationSuccessString = AddRequestUUID(reconciliationSuccessString, l.logRequestUUID) + l.zapLogger.Info(reconciliationSuccessString) + _, err = f.WriteString(reconciliationSuccessString) if err != nil { - return fmt.Errorf("failed to write reconciliation success string %s: %w", reconciliationSuccessString, err) + err = fmt.Errorf("failed to write reconciliation success string %s: %w", reconciliationSuccessString, err) + l.zapLogger.Error(err.Error()) + return err } return nil @@ -513,13 +540,15 @@ func (l *Logger) ReconcileFailureStream( os.FileMode(utils.DefaultFilePermissions), ) if err != nil { - return fmt.Errorf("failed to open file %s: %w", path.Join(l.logDir, reconcileFailureStreamFile), err) + err = fmt.Errorf("failed to open file %s: %w", path.Join(l.logDir, reconcileFailureStreamFile), err) + l.zapLogger.Error(err.Error()) + return err } defer closeFile(f) reconciliationFailureString := fmt.Sprintf( - "Type:%s Account: %s Currency: %s Block: %s:%d computed: %s live: %s, RequestUUID: %s\n", + "Type:%s Account: %s Currency: %s Block: %s:%d computed: %s live: %s", reconciliationType, types.AccountString(account), types.CurrencyString(currency), @@ -527,11 +556,14 @@ func (l *Logger) ReconcileFailureStream( block.Index, computedBalance, liveBalance, - l.logRequestUUID, ) + reconciliationFailureString = AddRequestUUID(reconciliationFailureString, l.logRequestUUID) + l.zapLogger.Info(reconciliationFailureString) _, err = f.WriteString(reconciliationFailureString) if err != nil { - return fmt.Errorf("failed to write reconciliation failure string %s: %w", reconciliationFailureString, err) + err = fmt.Errorf("failed to write reconciliation failure string %s: %w", reconciliationFailureString, err) + l.zapLogger.Error(err.Error()) + return err } return nil @@ -590,7 +622,7 @@ func LogTransactionCreated( func AddRequestUUIDFromContext(ctx context.Context, msg string) string { requestUUID := requestUUIDFromContext(ctx) if requestUUID != "" { - msg = fmt.Sprintf("%s,RequestUUID: %s\n", msg, requestUUID) + msg = fmt.Sprintf("%s, RequestUUID: %s", msg, requestUUID) } return msg } @@ -598,7 +630,7 @@ func AddRequestUUIDFromContext(ctx context.Context, msg string) string { // Add requestUUID to the tip func AddRequestUUID(msg string, requestUUID string) string { if requestUUID != "" { - msg = fmt.Sprintf("%s,RequestUUID: %s\n", msg, requestUUID) + msg = fmt.Sprintf("%s, RequestUUID: %s", msg, requestUUID) } return msg } From 9ce7a3bd73dc9ee0479aeab7d51a330325fa56ac Mon Sep 17 00:00:00 2001 From: Madhur Shrimal Date: Fri, 9 Dec 2022 14:18:19 -0500 Subject: [PATCH 363/376] update new version (#378) --- Dockerfile | 2 +- cmd/root.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index ae28c100..1545c72b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -31,7 +31,7 @@ RUN mkdir -p "$GOPATH/src" "$GOPATH/bin" && chmod -R 777 "$GOPATH" WORKDIR /go/src -ARG VERSION=v0.10.0 +ARG VERSION=v0.10.1 RUN git clone https://github.com/coinbase/rosetta-cli.git && \ cd rosetta-cli && \ git fetch --all --tags && \ diff --git a/cmd/root.go b/cmd/root.go index 6c9c3fc7..d396a598 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -454,6 +454,6 @@ var versionCmd = &cobra.Command{ Use: "version", Short: "Print rosetta-cli version", Run: func(cmd *cobra.Command, args []string) { - fmt.Println("v0.10.0") + fmt.Println("v0.10.1") }, } From 0b38f55a803e8fbf122e6a5c33732449bb3dece7 Mon Sep 17 00:00:00 2001 From: jingweicb <96205264+jingweicb@users.noreply.github.com> Date: Fri, 9 Dec 2022 14:47:29 -0800 Subject: [PATCH 364/376] make logs colorful, mainly cyan (#379) --- pkg/logger/logger.go | 50 ++++++++++++++++++++++---------------------- 1 file changed, 25 insertions(+), 25 deletions(-) diff --git a/pkg/logger/logger.go b/pkg/logger/logger.go index cd2aebe4..10ea9159 100644 --- a/pkg/logger/logger.go +++ b/pkg/logger/logger.go @@ -168,7 +168,7 @@ func (l *Logger) LogDataStatus(ctx context.Context, status *results.CheckDataSta } l.lastStatsMessage = statsMessage - l.zapLogger.Info(statsMessage) + color.Cyan(statsMessage) // If Progress is nil, it means we're already done. if status.Progress == nil { @@ -194,7 +194,7 @@ func (l *Logger) LogDataStatus(ctx context.Context, status *results.CheckDataSta } l.lastProgressMessage = progressMessage - l.zapLogger.Info(progressMessage) + color.Cyan(progressMessage) } // LogConstructionStatus logs results.CheckConstructionStatus. @@ -218,7 +218,7 @@ func (l *Logger) LogConstructionStatus( statsMessage = AddRequestUUID(statsMessage, l.logRequestUUID) l.lastStatsMessage = statsMessage - l.zapLogger.Info(statsMessage) + color.Cyan(statsMessage) } // LogMemoryStats logs memory usage information. @@ -252,7 +252,7 @@ func (l *Logger) AddBlockStream( ) if err != nil { err = fmt.Errorf("failed to open file %s: %w", path.Join(l.logDir, blockStreamFile), err) - l.zapLogger.Error(err.Error()) + color.Red(err.Error()) return err } @@ -267,7 +267,7 @@ func (l *Logger) AddBlockStream( block.ParentBlockIdentifier.Hash, ) blockString = AddRequestUUID(blockString, l.logRequestUUID) - l.zapLogger.Info(blockString) + color.Cyan(blockString) if _, err := f.WriteString(blockString); err != nil { return fmt.Errorf("failed to write block string %s: %w", blockString, err) } @@ -292,7 +292,7 @@ func (l *Logger) RemoveBlockStream( ) if err != nil { err = fmt.Errorf("failed to open file %s: %w", path.Join(l.logDir, blockStreamFile), err) - l.zapLogger.Error(err.Error()) + color.Red(err.Error()) return err } @@ -305,11 +305,11 @@ func (l *Logger) RemoveBlockStream( block.Hash, ) blockString = AddRequestUUID(blockString, l.logRequestUUID) - l.zapLogger.Info(blockString) + color.Cyan(blockString) _, err = f.WriteString(blockString) if err != nil { err = fmt.Errorf("failed to write block string %s: %w", blockString, err) - l.zapLogger.Error(err.Error()) + color.Red(err.Error()) return err } @@ -333,7 +333,7 @@ func (l *Logger) TransactionStream( ) if err != nil { err = fmt.Errorf("failed to open file %s: %w", path.Join(l.logDir, transactionStreamFile), err) - l.zapLogger.Error(err.Error()) + color.Red(err.Error()) return err } @@ -347,11 +347,11 @@ func (l *Logger) TransactionStream( block.BlockIdentifier.Hash, ) transactionString = AddRequestUUID(transactionString, l.logRequestUUID) - l.zapLogger.Info(transactionString) + color.Cyan(transactionString) _, err = f.WriteString(transactionString) if err != nil { err = fmt.Errorf("failed to write transaction string %s: %w", transactionString, err) - l.zapLogger.Error(err.Error()) + color.Red(err.Error()) return err } @@ -383,11 +383,11 @@ func (l *Logger) TransactionStream( *op.Status, ) transactionOperationString = AddRequestUUID(transactionOperationString, l.logRequestUUID) - l.zapLogger.Info(transactionOperationString) + color.Cyan(transactionOperationString) _, err = f.WriteString(transactionOperationString) if err != nil { err = fmt.Errorf("failed to write transaction operation string %s: %w", transactionOperationString, err) - l.zapLogger.Error(err.Error()) + color.Red(err.Error()) return err } } @@ -413,7 +413,7 @@ func (l *Logger) BalanceStream( ) if err != nil { err = fmt.Errorf("failed to open file %s: %w", path.Join(l.logDir, balanceStreamFile), err) - l.zapLogger.Error(err.Error()) + color.Red(err.Error()) return err } @@ -429,10 +429,10 @@ func (l *Logger) BalanceStream( balanceChange.Block.Hash, ) balanceLog = AddRequestUUID(balanceLog, l.logRequestUUID) - l.zapLogger.Info(balanceLog) + color.Cyan(balanceLog) if _, err := f.WriteString(fmt.Sprintf("%s\n", balanceLog)); err != nil { err = fmt.Errorf("failed to write balance log %s: %w", balanceLog, err) - l.zapLogger.Error(err.Error()) + color.Red(err.Error()) return err } } @@ -460,7 +460,7 @@ func (l *Logger) ReconcileSuccessStream( ) if err != nil { err = fmt.Errorf("failed to open file %s: %w", path.Join(l.logDir, reconcileSuccessStreamFile), err) - l.zapLogger.Error(err.Error()) + color.Red(err.Error()) return err } @@ -473,7 +473,7 @@ func (l *Logger) ReconcileSuccessStream( block.Index, ) reconciledLog = AddRequestUUID(reconciledLog, l.logRequestUUID) - l.zapLogger.Info(reconciledLog) + color.Cyan(reconciledLog) reconciliationSuccessString := fmt.Sprintf( "Type:%s Account: %s Currency: %s Balance: %s Block: %d:%s", @@ -485,12 +485,12 @@ func (l *Logger) ReconcileSuccessStream( block.Hash, ) reconciliationSuccessString = AddRequestUUID(reconciliationSuccessString, l.logRequestUUID) - l.zapLogger.Info(reconciliationSuccessString) + color.Cyan(reconciliationSuccessString) _, err = f.WriteString(reconciliationSuccessString) if err != nil { err = fmt.Errorf("failed to write reconciliation success string %s: %w", reconciliationSuccessString, err) - l.zapLogger.Error(err.Error()) + color.Red(err.Error()) return err } @@ -541,7 +541,7 @@ func (l *Logger) ReconcileFailureStream( ) if err != nil { err = fmt.Errorf("failed to open file %s: %w", path.Join(l.logDir, reconcileFailureStreamFile), err) - l.zapLogger.Error(err.Error()) + color.Red(err.Error()) return err } @@ -558,11 +558,11 @@ func (l *Logger) ReconcileFailureStream( liveBalance, ) reconciliationFailureString = AddRequestUUID(reconciliationFailureString, l.logRequestUUID) - l.zapLogger.Info(reconciliationFailureString) + color.Cyan(reconciliationFailureString) _, err = f.WriteString(reconciliationFailureString) if err != nil { err = fmt.Errorf("failed to write reconciliation failure string %s: %w", reconciliationFailureString, err) - l.zapLogger.Error(err.Error()) + color.Red(err.Error()) return err } @@ -622,7 +622,7 @@ func LogTransactionCreated( func AddRequestUUIDFromContext(ctx context.Context, msg string) string { requestUUID := requestUUIDFromContext(ctx) if requestUUID != "" { - msg = fmt.Sprintf("%s, RequestUUID: %s", msg, requestUUID) + msg = fmt.Sprintf("%s, RequestID: %s", msg, requestUUID) } return msg } @@ -630,7 +630,7 @@ func AddRequestUUIDFromContext(ctx context.Context, msg string) string { // Add requestUUID to the tip func AddRequestUUID(msg string, requestUUID string) string { if requestUUID != "" { - msg = fmt.Sprintf("%s, RequestUUID: %s", msg, requestUUID) + msg = fmt.Sprintf("%s, RequestID: %s", msg, requestUUID) } return msg } From 820f29f076f6a73768a290199562ad4397fc9d7a Mon Sep 17 00:00:00 2001 From: Xiaying Peng Date: Mon, 12 Dec 2022 14:39:17 -0800 Subject: [PATCH 365/376] enable status port --- cmd/root.go | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/cmd/root.go b/cmd/root.go index d396a598..e63c86d9 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -58,6 +58,7 @@ var ( inMemoryMode bool tableSize int64 requestUUID string + statusPort uint // Config is the populated *configuration.Configuration from // the configurationFile. If none is provided, this is set @@ -288,6 +289,13 @@ default values.`, "requestUUID configures the requestUUID in logs, which aims to enable search logs by requestUUID", ) + checkDataCmd.Flags().UintVar( + &statusPort, + "status-port", + 0, + "status-port configures the status query port, this will override the status_port", + ) + rootCmd.AddCommand(checkDataCmd) checkConstructionCmd.Flags().StringVar( &asserterConfigurationFile, @@ -419,6 +427,10 @@ func initConfig() { if len(requestUUID) != 0 { Config.RequestUUID = requestUUID } + + if statusPort > 0 { + Config.Data.StatusPort = statusPort + } } func ensureDataDirectoryExists() { From 5ab313064913b562ca14a828d933671a90f0e4a8 Mon Sep 17 00:00:00 2001 From: Xiaying Peng Date: Mon, 12 Dec 2022 14:45:11 -0800 Subject: [PATCH 366/376] Update root.go --- cmd/root.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/root.go b/cmd/root.go index e63c86d9..abac6568 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -58,7 +58,7 @@ var ( inMemoryMode bool tableSize int64 requestUUID string - statusPort uint + statusPort uint // Config is the populated *configuration.Configuration from // the configurationFile. If none is provided, this is set From 118347533fde413a1d1e93ef0e89fd903a476950 Mon Sep 17 00:00:00 2001 From: Xiaying Peng Date: Mon, 12 Dec 2022 14:52:53 -0800 Subject: [PATCH 367/376] Update root.go --- cmd/root.go | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/cmd/root.go b/cmd/root.go index abac6568..5742ee7f 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -332,6 +332,13 @@ default values.`, "requestUUID configures the requestUUID in logs, which aims to enable search logs by requestUUID", ) + checkConstructionCmd.Flags().UintVar( + &statusPort, + "status-port", + 0, + "status-port configures the status query port, this will override the status_port", + ) + rootCmd.AddCommand(checkConstructionCmd) // View Commands From 4a1468e0002f4cf20f45a6bf9146589974334f4b Mon Sep 17 00:00:00 2001 From: Xiaying Peng Date: Mon, 12 Dec 2022 15:14:42 -0800 Subject: [PATCH 368/376] err handling --- cmd/root.go | 3 +++ configuration/configuration.go | 3 +-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/cmd/root.go b/cmd/root.go index 5742ee7f..6e50a0db 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -437,6 +437,9 @@ func initConfig() { if statusPort > 0 { Config.Data.StatusPort = statusPort + if Config.Construction != nil { + Config.Construction.StatusPort = statusPort + } } } diff --git a/configuration/configuration.go b/configuration/configuration.go index e3200e37..0e4fd965 100644 --- a/configuration/configuration.go +++ b/configuration/configuration.go @@ -57,8 +57,7 @@ func DefaultPerfConfiguration() *CheckPerfConfiguration { } // DefaultConfiguration returns a *Configuration with the -// EthereumNetwork, DefaultURL, DefaultTimeout, -// DefaultConstructionConfiguration and DefaultDataConfiguration. +// EthereumNetwork, DefaultURL, DefaultTimeout, and DefaultDataConfiguration. func DefaultConfiguration() *Configuration { return &Configuration{ Network: EthereumNetwork, From 193721d3cd0b7d706fc9eba66f5209df9fff003a Mon Sep 17 00:00:00 2001 From: jingweicb <96205264+jingweicb@users.noreply.github.com> Date: Fri, 6 Jan 2023 17:48:20 -0800 Subject: [PATCH 369/376] init customized info (#381) --- cmd/check_construction.go | 1 + cmd/check_data.go | 1 + cmd/root.go | 12 ++++++ configuration/types.go | 4 ++ pkg/logger/logger.go | 77 +++++++++++++++++++++++++++++++++++++- pkg/tester/construction.go | 2 + pkg/tester/data.go | 3 ++ 7 files changed, 99 insertions(+), 1 deletion(-) diff --git a/cmd/check_construction.go b/cmd/check_construction.go index c05c6745..3371878e 100644 --- a/cmd/check_construction.go +++ b/cmd/check_construction.go @@ -149,6 +149,7 @@ func runCheckConstructionCmd(_ *cobra.Command, _ []string) error { g, ctx := errgroup.WithContext(ctx) ctx = logger.AddRequestUUIDToContext(ctx, Config.RequestUUID) + ctx = logger.AddInfoMetaDataToContext(ctx, Config.InfoMetaData) g.Go(func() error { return constructionTester.StartPeriodicLogger(ctx) diff --git a/cmd/check_data.go b/cmd/check_data.go index 81637654..5d338b40 100644 --- a/cmd/check_data.go +++ b/cmd/check_data.go @@ -156,6 +156,7 @@ func runCheckDataCmd(_ *cobra.Command, _ []string) error { g, ctx := errgroup.WithContext(ctx) ctx = logger.AddRequestUUIDToContext(ctx, Config.RequestUUID) + ctx = logger.AddInfoMetaDataToContext(ctx, Config.InfoMetaData) g.Go(func() error { return dataTester.StartPeriodicLogger(ctx) diff --git a/cmd/root.go b/cmd/root.go index 6e50a0db..747239ac 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -59,6 +59,7 @@ var ( tableSize int64 requestUUID string statusPort uint + InfoMetaData string // Config is the populated *configuration.Configuration from // the configurationFile. If none is provided, this is set @@ -289,6 +290,13 @@ default values.`, "requestUUID configures the requestUUID in logs, which aims to enable search logs by requestUUID", ) + checkDataCmd.Flags().StringVar( + &InfoMetaData, + "info-metadata", + "", + "metadata configures the metadata which aims to show in logs", + ) + checkDataCmd.Flags().UintVar( &statusPort, "status-port", @@ -441,6 +449,10 @@ func initConfig() { Config.Construction.StatusPort = statusPort } } + + if len(InfoMetaData) != 0 { + Config.InfoMetaData = InfoMetaData + } } func ensureDataDirectoryExists() { diff --git a/configuration/types.go b/configuration/types.go index b7b5d7bc..617d7100 100644 --- a/configuration/types.go +++ b/configuration/types.go @@ -461,6 +461,10 @@ type Configuration struct { // then this value must be true. CoinSupported bool `json:"coin_supported"` + // InfoMetaData is a map of key:value + // which aims to show in the log for search + InfoMetaData string `json:"info_metadata,omitempty"` + Construction *ConstructionConfiguration `json:"construction"` Data *DataConfiguration `json:"data"` Perf *CheckPerfConfiguration `json:"perf"` diff --git a/pkg/logger/logger.go b/pkg/logger/logger.go index 10ea9159..427d9a38 100644 --- a/pkg/logger/logger.go +++ b/pkg/logger/logger.go @@ -20,6 +20,7 @@ import ( "log" "os" "path" + "strings" "go.uber.org/zap" "go.uber.org/zap/zapcore" @@ -43,6 +44,8 @@ type contextKey int const ( RequestUUID contextKey = iota + InfoMetaDataKey contextKey = iota + // blockStreamFile contains the stream of processed // blocks and whether they were added or removed. blockStreamFile = "blocks.txt" @@ -83,6 +86,7 @@ type Logger struct { logBalanceChanges bool logReconciliation bool logRequestUUID string + logInfoMetaData map[string]string lastStatsMessage string lastProgressMessage string @@ -100,6 +104,7 @@ func NewLogger( checkType CheckType, network *types.NetworkIdentifier, logRequestUUID string, + logInfoMetaData map[string]string, fields ...zap.Field, ) (*Logger, error) { zapLogger, err := buildZapLogger(checkType, network, logRequestUUID, fields...) @@ -113,6 +118,7 @@ func NewLogger( logBalanceChanges: logBalanceChanges, logReconciliation: logReconciliation, logRequestUUID: logRequestUUID, + logInfoMetaData: logInfoMetaData, zapLogger: zapLogger, }, nil } @@ -161,6 +167,7 @@ func (l *Logger) LogDataStatus(ctx context.Context, status *results.CheckDataSta ) statsMessage = AddRequestUUID(statsMessage, l.logRequestUUID) + statsMessage = AddInfoMetaData(statsMessage, l.logInfoMetaData) // Don't print out the same stats message twice. if statsMessage == l.lastStatsMessage { @@ -187,6 +194,7 @@ func (l *Logger) LogDataStatus(ctx context.Context, status *results.CheckDataSta ) progressMessage = AddRequestUUID(progressMessage, l.logRequestUUID) + progressMessage = AddInfoMetaData(progressMessage, l.logInfoMetaData) // Don't print out the same progress message twice. if progressMessage == l.lastProgressMessage { @@ -216,7 +224,8 @@ func (l *Logger) LogConstructionStatus( } statsMessage = AddRequestUUID(statsMessage, l.logRequestUUID) - + statsMessage = AddInfoMetaData(statsMessage, l.logInfoMetaData) + l.lastStatsMessage = statsMessage color.Cyan(statsMessage) } @@ -232,6 +241,7 @@ func LogMemoryStats(ctx context.Context) { memUsage.GarbageCollections, ) statsMessage = AddRequestUUIDFromContext(ctx, statsMessage) + statsMessage = AddInfoMetaDataFromContext(ctx, statsMessage) color.Cyan(statsMessage) } @@ -267,6 +277,7 @@ func (l *Logger) AddBlockStream( block.ParentBlockIdentifier.Hash, ) blockString = AddRequestUUID(blockString, l.logRequestUUID) + blockString = AddInfoMetaData(blockString, l.logInfoMetaData) color.Cyan(blockString) if _, err := f.WriteString(blockString); err != nil { return fmt.Errorf("failed to write block string %s: %w", blockString, err) @@ -305,6 +316,7 @@ func (l *Logger) RemoveBlockStream( block.Hash, ) blockString = AddRequestUUID(blockString, l.logRequestUUID) + blockString = AddInfoMetaData(blockString, l.logInfoMetaData) color.Cyan(blockString) _, err = f.WriteString(blockString) if err != nil { @@ -347,6 +359,7 @@ func (l *Logger) TransactionStream( block.BlockIdentifier.Hash, ) transactionString = AddRequestUUID(transactionString, l.logRequestUUID) + transactionString = AddInfoMetaData(transactionString, l.logInfoMetaData) color.Cyan(transactionString) _, err = f.WriteString(transactionString) if err != nil { @@ -383,6 +396,7 @@ func (l *Logger) TransactionStream( *op.Status, ) transactionOperationString = AddRequestUUID(transactionOperationString, l.logRequestUUID) + transactionOperationString = AddInfoMetaData(transactionOperationString, l.logInfoMetaData) color.Cyan(transactionOperationString) _, err = f.WriteString(transactionOperationString) if err != nil { @@ -429,6 +443,7 @@ func (l *Logger) BalanceStream( balanceChange.Block.Hash, ) balanceLog = AddRequestUUID(balanceLog, l.logRequestUUID) + balanceLog = AddInfoMetaData(balanceLog, l.logInfoMetaData) color.Cyan(balanceLog) if _, err := f.WriteString(fmt.Sprintf("%s\n", balanceLog)); err != nil { err = fmt.Errorf("failed to write balance log %s: %w", balanceLog, err) @@ -473,6 +488,7 @@ func (l *Logger) ReconcileSuccessStream( block.Index, ) reconciledLog = AddRequestUUID(reconciledLog, l.logRequestUUID) + reconciledLog = AddInfoMetaData(reconciledLog, l.logInfoMetaData) color.Cyan(reconciledLog) reconciliationSuccessString := fmt.Sprintf( @@ -485,6 +501,7 @@ func (l *Logger) ReconcileSuccessStream( block.Hash, ) reconciliationSuccessString = AddRequestUUID(reconciliationSuccessString, l.logRequestUUID) + reconciliationSuccessString = AddInfoMetaData(reconciliationSuccessString, l.logInfoMetaData) color.Cyan(reconciliationSuccessString) _, err = f.WriteString(reconciliationSuccessString) @@ -558,6 +575,7 @@ func (l *Logger) ReconcileFailureStream( liveBalance, ) reconciliationFailureString = AddRequestUUID(reconciliationFailureString, l.logRequestUUID) + reconciliationFailureString = AddInfoMetaData(reconciliationFailureString, l.logInfoMetaData) color.Cyan(reconciliationFailureString) _, err = f.WriteString(reconciliationFailureString) if err != nil { @@ -649,3 +667,60 @@ func requestUUIDFromContext(ctx context.Context) string { return "" } } + +// Add InfoMetaData k-v pairs to the tip +func AddInfoMetaDataFromContext(ctx context.Context, msg string) string { + logInfoMetaData := InfoMetaDataFromContext(ctx) + if len(logInfoMetaData) != 0 { + for k, v := range logInfoMetaData { + msg = fmt.Sprintf("%s, %s: %s", msg, k, v) + } + } + return msg +} + +// Add InfoMetaData k-v pairs to the tip +func AddInfoMetaData(msg string, logInfoMetaData map[string]string) string { + if len(logInfoMetaData) != 0 { + for k, v := range logInfoMetaData { + msg = fmt.Sprintf("%s, %s: %s", msg, k, v) + } + } + return msg +} + +// AddInfoMetaDataToContext will add InfoMetaData to the context, and return the new context +func AddInfoMetaDataToContext(ctx context.Context, InfoMetaData string) context.Context { + return context.WithValue(ctx, InfoMetaDataKey, InfoMetaData) +} + +// InfoMetaDataFromContext is used to extract InfoMetaData from a context +func InfoMetaDataFromContext(ctx context.Context) map[string]string { + var metadata string + switch v := ctx.Value(InfoMetaDataKey).(type) { + case string: + metadata = v + default: + metadata = "" + } + return ConvertStringToMap(metadata) +} + +// ConvertStringToMap is used to convert a string to map by split , and ; +func ConvertStringToMap(metadata string) map[string]string { + InfoMetaDataMap := make(map[string]string) + if len(metadata) == 0 { + return InfoMetaDataMap + } + pairs := strings.Split(metadata, ",") + for _, pair := range pairs { + kv := strings.Split(pair, ":") + if(len(kv) != 2) { + log := fmt.Sprintf("the %s from %s could be transfer to key value pair", pair, metadata) + color.Yellow(log) + } else { + InfoMetaDataMap[strings.TrimSpace(kv[0])] = strings.TrimSpace(kv[1]) + } + } + return InfoMetaDataMap +} diff --git a/pkg/tester/construction.go b/pkg/tester/construction.go index ba8a806f..ad26185c 100644 --- a/pkg/tester/construction.go +++ b/pkg/tester/construction.go @@ -115,6 +115,7 @@ func InitializeConstruction( } counterStorage := modules.NewCounterStorage(localStore) + logInfoMetaData := logger.ConvertStringToMap(config.InfoMetaData) logger, err := logger.NewLogger( dataPath, false, @@ -124,6 +125,7 @@ func InitializeConstruction( logger.Construction, network, config.RequestUUID, + logInfoMetaData, ) if err != nil { return nil, fmt.Errorf("unable to initialize logger with error: %w", err) diff --git a/pkg/tester/data.go b/pkg/tester/data.go index fbf4d189..b687e885 100644 --- a/pkg/tester/data.go +++ b/pkg/tester/data.go @@ -232,6 +232,7 @@ func InitializeData( blockStorage := modules.NewBlockStorage(localStore, config.SerialBlockWorkers) balanceStorage := modules.NewBalanceStorage(localStore) + logInfoMetaData := logger.ConvertStringToMap(config.InfoMetaData) logger, err := logger.NewLogger( dataPath, config.Data.LogBlocks, @@ -241,6 +242,7 @@ func InitializeData( logger.Data, network, config.RequestUUID, + logInfoMetaData, ) if err != nil { return nil, fmt.Errorf("unable to initialize logger with error: %w", err) @@ -1089,6 +1091,7 @@ func (t *DataTester) recursiveOpSearch( logger.Data, t.network, EmptyRequestUUID, + nil, ) if err != nil { From f476c00ab1755f94adc3668e323ed948fd378f4b Mon Sep 17 00:00:00 2001 From: jingweicb <96205264+jingweicb@users.noreply.github.com> Date: Mon, 16 Jan 2023 12:27:09 -0800 Subject: [PATCH 370/376] add customized info/metadata (#382) --- cmd/check_construction.go | 30 +++-- cmd/check_data.go | 26 +++- configuration/types.go | 10 +- go.mod | 3 +- go.sum | 10 +- pkg/logger/logger.go | 175 +++++++++++++-------------- pkg/tester/construction.go | 110 ++++++++++++----- pkg/tester/data.go | 236 +++++++++++++++++++++++++++---------- 8 files changed, 396 insertions(+), 204 deletions(-) diff --git a/cmd/check_construction.go b/cmd/check_construction.go index 3371878e..933f5797 100644 --- a/cmd/check_construction.go +++ b/cmd/check_construction.go @@ -21,6 +21,7 @@ import ( cliErrs "github.com/coinbase/rosetta-cli/pkg/errors" "github.com/coinbase/rosetta-cli/pkg/logger" + "github.com/fatih/color" "github.com/coinbase/rosetta-cli/pkg/results" "github.com/coinbase/rosetta-cli/pkg/tester" @@ -54,6 +55,7 @@ and UTXO-based blockchains). However, we plan to add support for testing arbitrary scenarios (for example, staking and governance).`, RunE: runCheckConstructionCmd, } + constructionMetadata string ) func runCheckConstructionCmd(_ *cobra.Command, _ []string) error { @@ -66,6 +68,10 @@ func runCheckConstructionCmd(_ *cobra.Command, _ []string) error { ) } + metadataMap := logger.ConvertStringToMap(Config.InfoMetaData) + metadataMap = logger.AddRequestUUIDToMap(metadataMap, Config.RequestUUID) + constructionMetadata = logger.ConvertMapToString(metadataMap) + ensureDataDirectoryExists() ctx, cancel := context.WithCancel(Context) @@ -74,6 +80,7 @@ func runCheckConstructionCmd(_ *cobra.Command, _ []string) error { fetcher.WithRetryElapsedTime(time.Duration(Config.RetryElapsedTime) * time.Second), fetcher.WithTimeout(time.Duration(Config.HTTPTimeout) * time.Second), fetcher.WithMaxRetries(Config.MaxRetries), + fetcher.WithMetaData(metadata), } if Config.ForceRetry { fetcherOpts = append(fetcherOpts, fetcher.WithForceRetry()) @@ -87,22 +94,26 @@ func runCheckConstructionCmd(_ *cobra.Command, _ []string) error { _, _, fetchErr := fetcher.InitializeAsserter(ctx, Config.Network, Config.ValidationFile) if fetchErr != nil { cancel() + err := fmt.Errorf("unable to initialize asserter for fetcher: %w%s", fetchErr.Err, constructionMetadata) + color.Red(err.Error()) return results.ExitConstruction( Config, nil, nil, - fmt.Errorf("unable to initialize asserter for fetcher: %w", fetchErr.Err), + err, ) } _, err := utils.CheckNetworkSupported(ctx, Config.Network, fetcher) if err != nil { cancel() + err = fmt.Errorf("unable to confirm network %s is supported: %w%s", types.PrintStruct(Config.Network), err, constructionMetadata) + color.Red(err.Error()) return results.ExitConstruction( Config, nil, nil, - fmt.Errorf("unable to confirm network %s is supported: %w", types.PrintStruct(Config.Network), err), + err, ) } @@ -111,11 +122,13 @@ func runCheckConstructionCmd(_ *cobra.Command, _ []string) error { ctx, fetcher, Config.Network, asserterConfigurationFile, ); err != nil { cancel() + err = fmt.Errorf("network options don't match asserter configuration file %s: %w%s", asserterConfigurationFile, err, constructionMetadata) + color.Red(err.Error()) return results.ExitConstruction( Config, nil, nil, - fmt.Errorf("network options don't match asserter configuration file %s: %w", asserterConfigurationFile, err), + err, ) } } @@ -129,27 +142,30 @@ func runCheckConstructionCmd(_ *cobra.Command, _ []string) error { &SignalReceived, ) if err != nil { + err = fmt.Errorf("unable to initialize construction tester: %w%s", err, constructionMetadata) + color.Red(err.Error()) return results.ExitConstruction( Config, nil, nil, - fmt.Errorf("unable to initialize construction tester: %w", err), + err, ) } defer constructionTester.CloseDatabase(ctx) if err := constructionTester.PerformBroadcasts(ctx); err != nil { + err = fmt.Errorf("unable to perform broadcasts: %w%s", err, constructionMetadata) + color.Red(err.Error()) return results.ExitConstruction( Config, nil, nil, - fmt.Errorf("unable to perform broadcasts: %w", err), + err, ) } g, ctx := errgroup.WithContext(ctx) - ctx = logger.AddRequestUUIDToContext(ctx, Config.RequestUUID) - ctx = logger.AddInfoMetaDataToContext(ctx, Config.InfoMetaData) + ctx = logger.AddMetadataMapToContext(ctx, metadataMap) g.Go(func() error { return constructionTester.StartPeriodicLogger(ctx) diff --git a/cmd/check_data.go b/cmd/check_data.go index 5d338b40..026285ea 100644 --- a/cmd/check_data.go +++ b/cmd/check_data.go @@ -25,6 +25,7 @@ import ( "github.com/coinbase/rosetta-sdk-go/fetcher" "github.com/coinbase/rosetta-sdk-go/types" "github.com/coinbase/rosetta-sdk-go/utils" + "github.com/fatih/color" "github.com/spf13/cobra" "golang.org/x/sync/errgroup" ) @@ -69,17 +70,23 @@ bootstrap balance config. You can look at the examples folder for an example of what one of these files looks like.`, RunE: runCheckDataCmd, } + metadata string ) func runCheckDataCmd(_ *cobra.Command, _ []string) error { ensureDataDirectoryExists() ctx, cancel := context.WithCancel(Context) + metadataMap := logger.ConvertStringToMap(Config.InfoMetaData) + metadataMap = logger.AddRequestUUIDToMap(metadataMap, Config.RequestUUID) + metadata = logger.ConvertMapToString(metadataMap) + fetcherOpts := []fetcher.Option{ fetcher.WithMaxConnections(Config.MaxOnlineConnections), fetcher.WithRetryElapsedTime(time.Duration(Config.RetryElapsedTime) * time.Second), fetcher.WithTimeout(time.Duration(Config.HTTPTimeout) * time.Second), fetcher.WithMaxRetries(Config.MaxRetries), + fetcher.WithMetaData(metadata), } if Config.ForceRetry { fetcherOpts = append(fetcherOpts, fetcher.WithForceRetry()) @@ -93,11 +100,13 @@ func runCheckDataCmd(_ *cobra.Command, _ []string) error { _, _, fetchErr := fetcher.InitializeAsserter(ctx, Config.Network, Config.ValidationFile) if fetchErr != nil { cancel() + err := fmt.Errorf("unable to initialize asserter for fetcher: %w%s", fetchErr.Err, metadata) + color.Red(err.Error()) return results.ExitData( Config, nil, nil, - fmt.Errorf("unable to initialize asserter for fetcher: %w", fetchErr.Err), + err, "", "", ) @@ -106,11 +115,13 @@ func runCheckDataCmd(_ *cobra.Command, _ []string) error { networkStatus, err := utils.CheckNetworkSupported(ctx, Config.Network, fetcher) if err != nil { cancel() + err = fmt.Errorf("unable to confirm network %s is supported: %w%s", types.PrintStruct(Config.Network), err, metadata) + color.Red(err.Error()) return results.ExitData( Config, nil, nil, - fmt.Errorf("unable to confirm network %s is supported: %w", types.PrintStruct(Config.Network), err), + err, "", "", ) @@ -121,11 +132,13 @@ func runCheckDataCmd(_ *cobra.Command, _ []string) error { ctx, fetcher, Config.Network, asserterConfigurationFile, ); err != nil { cancel() + err = fmt.Errorf("network options don't match asserter configuration file %s: %w%s", asserterConfigurationFile, err, metadata) + color.Red(err.Error()) return results.ExitData( Config, nil, nil, - fmt.Errorf("network options don't match asserter configuration file %s: %w", asserterConfigurationFile, err), + err, "", "", ) @@ -143,11 +156,13 @@ func runCheckDataCmd(_ *cobra.Command, _ []string) error { &SignalReceived, ) if err != nil { + err = fmt.Errorf("unable to initialize data tester: %w%s", err, metadata) + color.Red(err.Error()) return results.ExitData( Config, nil, nil, - fmt.Errorf("unable to initialize data tester: %w", err), + err, "", "", ) @@ -155,8 +170,7 @@ func runCheckDataCmd(_ *cobra.Command, _ []string) error { defer dataTester.CloseDatabase(ctx) g, ctx := errgroup.WithContext(ctx) - ctx = logger.AddRequestUUIDToContext(ctx, Config.RequestUUID) - ctx = logger.AddInfoMetaDataToContext(ctx, Config.InfoMetaData) + ctx = logger.AddMetadataMapToContext(ctx, metadataMap) g.Go(func() error { return dataTester.StartPeriodicLogger(ctx) diff --git a/configuration/types.go b/configuration/types.go index 617d7100..01763a04 100644 --- a/configuration/types.go +++ b/configuration/types.go @@ -461,8 +461,14 @@ type Configuration struct { // then this value must be true. CoinSupported bool `json:"coin_supported"` - // InfoMetaData is a map of key:value - // which aims to show in the log for search + // InfoMetaData is a string, rosetta-cli will convert it into a map[string]string + // key-value are separated by ":" + // different key-value pairs are separated by "," + // an example: if users want to record "instance_name" as "1234", and "blockchain_name" as "Bitcoin", + // this field would be "instance_name:1234,blockchain_name:Bitcoin" + // if adding spaces before and after ":" and ",", it will be trimmed when building map + // " instance_name : xxxx , blockchain_name : xxxx " will be recorded same as + // "instance_name:xxxx,blockchain_name:xxxx" InfoMetaData string `json:"info_metadata,omitempty"` Construction *ConstructionConfiguration `json:"construction"` diff --git a/go.mod b/go.mod index fdfc466e..d39a1e3e 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,8 @@ module github.com/coinbase/rosetta-cli go 1.16 require ( - github.com/coinbase/rosetta-sdk-go v0.8.1 + github.com/coinbase/rosetta-sdk-go v0.8.3-0.20230113233127-4c691644d82a + github.com/coinbase/rosetta-sdk-go/types v1.0.0 github.com/fatih/color v1.13.0 github.com/google/go-cmp v0.5.6 // indirect github.com/mattn/go-colorable v0.1.12 // indirect diff --git a/go.sum b/go.sum index 2d23f0e9..190c8492 100644 --- a/go.sum +++ b/go.sum @@ -98,8 +98,10 @@ github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDk github.com/cloudflare/cloudflare-go v0.14.0/go.mod h1:EnwdgGMaFOruiPZRFSgn+TsQ3hQ7C/YWzIGLeu5c304= github.com/coinbase/kryptology v1.8.0 h1:Aoq4gdTsJhSU3lNWsD5BWmFSz2pE0GlmrljaOxepdYY= github.com/coinbase/kryptology v1.8.0/go.mod h1:RYXOAPdzOGUe3qlSFkMGn58i3xUA8hmxYHksuq+8ciI= -github.com/coinbase/rosetta-sdk-go v0.8.1 h1:WE+Temc8iz7Ra7sCpV9ymBJx78vItqFJ2xcSiPet1Pc= -github.com/coinbase/rosetta-sdk-go v0.8.1/go.mod h1:tXPR6AIW9ogsH4tYIaFOKOgfJNanCvcyl7JKLd4DToc= +github.com/coinbase/rosetta-sdk-go v0.8.3-0.20230113233127-4c691644d82a h1:Bym3+QUv9zIij91/4yX3hs5E2TsDOJ6+8egOeF8KBYI= +github.com/coinbase/rosetta-sdk-go v0.8.3-0.20230113233127-4c691644d82a/go.mod h1:ChOHc+BNq7zqJDDkui0DA124GOvlAiRbdgAc1U9GMDQ= +github.com/coinbase/rosetta-sdk-go/types v1.0.0 h1:jpVIwLcPoOeCR6o1tU+Xv7r5bMONNbHU7MuEHboiFuA= +github.com/coinbase/rosetta-sdk-go/types v1.0.0/go.mod h1:eq7W2TMRH22GTW0N0beDnN931DW0/WOI1R2sdHNHG4c= github.com/consensys/bavard v0.1.8-0.20210406032232-f3452dc9b572/go.mod h1:Bpd0/3mZuaj6Sj+PqrmIquiOKy397AKGThQPaGzNXAQ= github.com/consensys/bavard v0.1.8-0.20210915155054-088da2f7f54a/go.mod h1:9ItSMtA/dXMAiL7BG6bqW2m3NdSEObYWoH223nGHukI= github.com/consensys/gnark-crypto v0.4.1-0.20210426202927-39ac3d4b3f1f/go.mod h1:815PAHg3wvysy0SyIqanF8gZ0Y1wjk/hrDHD/iT88+Q= @@ -328,8 +330,8 @@ github.com/mimoo/StrobeGo v0.0.0-20181016162300-f8f6d4d2b643/go.mod h1:43+3pMjjK github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/mitchellh/mapstructure v1.4.1/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= -github.com/mitchellh/mapstructure v1.4.3 h1:OVowDSCllw/YjdLkam3/sm7wEtOy59d8ndGgCcyj8cs= -github.com/mitchellh/mapstructure v1.4.3/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= +github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= +github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/mitchellh/pointerstructure v1.2.0/go.mod h1:BRAsLI5zgXmw97Lf6s25bs8ohIXc3tViBH44KcwB2g4= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= diff --git a/pkg/logger/logger.go b/pkg/logger/logger.go index 427d9a38..5ade4f34 100644 --- a/pkg/logger/logger.go +++ b/pkg/logger/logger.go @@ -44,7 +44,7 @@ type contextKey int const ( RequestUUID contextKey = iota - InfoMetaDataKey contextKey = iota + MetadataMapKey contextKey = iota // blockStreamFile contains the stream of processed // blocks and whether they were added or removed. @@ -85,8 +85,7 @@ type Logger struct { logTransactions bool logBalanceChanges bool logReconciliation bool - logRequestUUID string - logInfoMetaData map[string]string + logMetadataMap map[string]string lastStatsMessage string lastProgressMessage string @@ -103,11 +102,10 @@ func NewLogger( logReconciliation bool, checkType CheckType, network *types.NetworkIdentifier, - logRequestUUID string, - logInfoMetaData map[string]string, + logMetadataMap map[string]string, fields ...zap.Field, ) (*Logger, error) { - zapLogger, err := buildZapLogger(checkType, network, logRequestUUID, fields...) + zapLogger, err := buildZapLogger(checkType, network, fields...) if err != nil { return nil, fmt.Errorf("failed to build zap logger: %w", err) } @@ -117,8 +115,7 @@ func NewLogger( logTransactions: logTransactions, logBalanceChanges: logBalanceChanges, logReconciliation: logReconciliation, - logRequestUUID: logRequestUUID, - logInfoMetaData: logInfoMetaData, + logMetadataMap: logMetadataMap, zapLogger: zapLogger, }, nil } @@ -126,7 +123,6 @@ func NewLogger( func buildZapLogger( checkType CheckType, network *types.NetworkIdentifier, - requestUUID string, fields ...zap.Field, ) (*zap.Logger, error) { config := zap.NewProductionConfig() @@ -135,7 +131,6 @@ func buildZapLogger( baseSlice := []zap.Field{ zap.String("blockchain", network.Blockchain), zap.String("network", network.Network), - zap.String("requestUUID", requestUUID), zap.String("check_type", string(checkType)), } mergedSlice := append(baseSlice, fields...) @@ -166,8 +161,7 @@ func (l *Logger) LogDataStatus(ctx context.Context, status *results.CheckDataSta status.Stats.ReconciliationCoverage*utils.OneHundred, ) - statsMessage = AddRequestUUID(statsMessage, l.logRequestUUID) - statsMessage = AddInfoMetaData(statsMessage, l.logInfoMetaData) + statsMessage = AddMetadata(statsMessage, l.logMetadataMap) // Don't print out the same stats message twice. if statsMessage == l.lastStatsMessage { @@ -193,8 +187,7 @@ func (l *Logger) LogDataStatus(ctx context.Context, status *results.CheckDataSta status.Progress.ReconcilerLastIndex, ) - progressMessage = AddRequestUUID(progressMessage, l.logRequestUUID) - progressMessage = AddInfoMetaData(progressMessage, l.logInfoMetaData) + progressMessage = AddMetadata(progressMessage, l.logMetadataMap) // Don't print out the same progress message twice. if progressMessage == l.lastProgressMessage { @@ -223,8 +216,7 @@ func (l *Logger) LogConstructionStatus( return } - statsMessage = AddRequestUUID(statsMessage, l.logRequestUUID) - statsMessage = AddInfoMetaData(statsMessage, l.logInfoMetaData) + statsMessage = AddMetadata(statsMessage, l.logMetadataMap) l.lastStatsMessage = statsMessage color.Cyan(statsMessage) @@ -240,8 +232,7 @@ func LogMemoryStats(ctx context.Context) { memUsage.System, memUsage.GarbageCollections, ) - statsMessage = AddRequestUUIDFromContext(ctx, statsMessage) - statsMessage = AddInfoMetaDataFromContext(ctx, statsMessage) + statsMessage = AddMetadataMapFromContext(ctx, statsMessage) color.Cyan(statsMessage) } @@ -276,8 +267,7 @@ func (l *Logger) AddBlockStream( block.ParentBlockIdentifier.Index, block.ParentBlockIdentifier.Hash, ) - blockString = AddRequestUUID(blockString, l.logRequestUUID) - blockString = AddInfoMetaData(blockString, l.logInfoMetaData) + blockString = AddMetadata(blockString, l.logMetadataMap) color.Cyan(blockString) if _, err := f.WriteString(blockString); err != nil { return fmt.Errorf("failed to write block string %s: %w", blockString, err) @@ -315,8 +305,7 @@ func (l *Logger) RemoveBlockStream( block.Index, block.Hash, ) - blockString = AddRequestUUID(blockString, l.logRequestUUID) - blockString = AddInfoMetaData(blockString, l.logInfoMetaData) + blockString = AddMetadata(blockString, l.logMetadataMap) color.Cyan(blockString) _, err = f.WriteString(blockString) if err != nil { @@ -358,8 +347,7 @@ func (l *Logger) TransactionStream( block.BlockIdentifier.Index, block.BlockIdentifier.Hash, ) - transactionString = AddRequestUUID(transactionString, l.logRequestUUID) - transactionString = AddInfoMetaData(transactionString, l.logInfoMetaData) + transactionString = AddMetadata(transactionString, l.logMetadataMap) color.Cyan(transactionString) _, err = f.WriteString(transactionString) if err != nil { @@ -395,8 +383,7 @@ func (l *Logger) TransactionStream( symbol, *op.Status, ) - transactionOperationString = AddRequestUUID(transactionOperationString, l.logRequestUUID) - transactionOperationString = AddInfoMetaData(transactionOperationString, l.logInfoMetaData) + transactionOperationString = AddMetadata(transactionOperationString, l.logMetadataMap) color.Cyan(transactionOperationString) _, err = f.WriteString(transactionOperationString) if err != nil { @@ -442,8 +429,7 @@ func (l *Logger) BalanceStream( balanceChange.Block.Index, balanceChange.Block.Hash, ) - balanceLog = AddRequestUUID(balanceLog, l.logRequestUUID) - balanceLog = AddInfoMetaData(balanceLog, l.logInfoMetaData) + balanceLog = AddMetadata(balanceLog, l.logMetadataMap) color.Cyan(balanceLog) if _, err := f.WriteString(fmt.Sprintf("%s\n", balanceLog)); err != nil { err = fmt.Errorf("failed to write balance log %s: %w", balanceLog, err) @@ -487,8 +473,7 @@ func (l *Logger) ReconcileSuccessStream( types.AccountString(account), block.Index, ) - reconciledLog = AddRequestUUID(reconciledLog, l.logRequestUUID) - reconciledLog = AddInfoMetaData(reconciledLog, l.logInfoMetaData) + reconciledLog = AddMetadata(reconciledLog, l.logMetadataMap) color.Cyan(reconciledLog) reconciliationSuccessString := fmt.Sprintf( @@ -500,8 +485,7 @@ func (l *Logger) ReconcileSuccessStream( block.Index, block.Hash, ) - reconciliationSuccessString = AddRequestUUID(reconciliationSuccessString, l.logRequestUUID) - reconciliationSuccessString = AddInfoMetaData(reconciliationSuccessString, l.logInfoMetaData) + reconciliationSuccessString = AddMetadata(reconciliationSuccessString, l.logMetadataMap) color.Cyan(reconciliationSuccessString) _, err = f.WriteString(reconciliationSuccessString) @@ -574,8 +558,7 @@ func (l *Logger) ReconcileFailureStream( computedBalance, liveBalance, ) - reconciliationFailureString = AddRequestUUID(reconciliationFailureString, l.logRequestUUID) - reconciliationFailureString = AddInfoMetaData(reconciliationFailureString, l.logInfoMetaData) + reconciliationFailureString = AddMetadata(reconciliationFailureString, l.logMetadataMap) color.Cyan(reconciliationFailureString) _, err = f.WriteString(reconciliationFailureString) if err != nil { @@ -617,6 +600,19 @@ func (l *Logger) Fatal(msg string, fields ...zap.Field) { l.zapLogger.Fatal(msg, fields...) } +// return a string of metadata +func (l *Logger) GetMetadata() string{ + metadatMap := l.logMetadataMap + metadata := ConvertMapToString(metadatMap) + return metadata +} + +// return a map of metadatMap +func (l *Logger) GetMetadataMap() map[string]string{ + metadatMap := l.logMetadataMap + return metadatMap +} + // Helper function to close log file func closeFile(f *os.File) { err := f.Close() @@ -636,81 +632,53 @@ func LogTransactionCreated( ) } -// Add requestUUID to the tip -func AddRequestUUIDFromContext(ctx context.Context, msg string) string { - requestUUID := requestUUIDFromContext(ctx) - if requestUUID != "" { - msg = fmt.Sprintf("%s, RequestID: %s", msg, requestUUID) - } - return msg -} - -// Add requestUUID to the tip -func AddRequestUUID(msg string, requestUUID string) string { - if requestUUID != "" { - msg = fmt.Sprintf("%s, RequestID: %s", msg, requestUUID) - } - return msg -} - -// AddRequestUUIDToContext will add a requestUUIDto the context, and return the new context -func AddRequestUUIDToContext(ctx context.Context, uuid string) context.Context { - return context.WithValue(ctx, RequestUUID, uuid) -} - -// requestUUIDFromContext is used to extract a request UUID from a context -func requestUUIDFromContext(ctx context.Context) string { - switch v := ctx.Value(RequestUUID).(type) { - case string: - return v - default: - return "" - } -} - // Add InfoMetaData k-v pairs to the tip -func AddInfoMetaDataFromContext(ctx context.Context, msg string) string { - logInfoMetaData := InfoMetaDataFromContext(ctx) - if len(logInfoMetaData) != 0 { - for k, v := range logInfoMetaData { - msg = fmt.Sprintf("%s, %s: %s", msg, k, v) +func AddMetadataMapFromContext(ctx context.Context, msg string) string { + metadataMap := metadataMapFromContext(ctx) + if len(metadataMap) != 0 { + for k, v := range metadataMap { + if len(k) != 0 && len(v) != 0 { + msg = fmt.Sprintf("%s, %s: %s", msg, k, v) + } } } return msg } -// Add InfoMetaData k-v pairs to the tip -func AddInfoMetaData(msg string, logInfoMetaData map[string]string) string { - if len(logInfoMetaData) != 0 { - for k, v := range logInfoMetaData { - msg = fmt.Sprintf("%s, %s: %s", msg, k, v) +// AddMetadataMapToContext will add InfoMetaData to the context, and return the new context +func AddMetadataMapToContext(ctx context.Context, metadataMap map[string]string) context.Context { + return context.WithValue(ctx, MetadataMapKey, metadataMap) +} + +// AddMetadata k-v pairs to the tip +func AddMetadata(msg string, metadataMap map[string]string) string { + if len(metadataMap) != 0 { + for k, v := range metadataMap { + if len(k) != 0 && len(v) != 0 { + msg = fmt.Sprintf("%s, %s: %s", msg, k, v) + } } } return msg } -// AddInfoMetaDataToContext will add InfoMetaData to the context, and return the new context -func AddInfoMetaDataToContext(ctx context.Context, InfoMetaData string) context.Context { - return context.WithValue(ctx, InfoMetaDataKey, InfoMetaData) -} - -// InfoMetaDataFromContext is used to extract InfoMetaData from a context -func InfoMetaDataFromContext(ctx context.Context) map[string]string { - var metadata string - switch v := ctx.Value(InfoMetaDataKey).(type) { - case string: - metadata = v +// metadataMapFromContext is used to extract metadataMap from a context +func metadataMapFromContext(ctx context.Context) map[string]string { + var metadataMap map[string]string + switch v := ctx.Value(MetadataMapKey).(type) { + case map[string]string: + metadataMap = v default: - metadata = "" + metadataMap = nil } - return ConvertStringToMap(metadata) + return metadataMap } // ConvertStringToMap is used to convert a string to map by split , and ; func ConvertStringToMap(metadata string) map[string]string { - InfoMetaDataMap := make(map[string]string) + metadataMap := make(map[string]string) if len(metadata) == 0 { - return InfoMetaDataMap + return metadataMap } pairs := strings.Split(metadata, ",") for _, pair := range pairs { @@ -719,8 +687,29 @@ func ConvertStringToMap(metadata string) map[string]string { log := fmt.Sprintf("the %s from %s could be transfer to key value pair", pair, metadata) color.Yellow(log) } else { - InfoMetaDataMap[strings.TrimSpace(kv[0])] = strings.TrimSpace(kv[1]) + metadataMap[strings.TrimSpace(kv[0])] = strings.TrimSpace(kv[1]) + } + } + return metadataMap +} + +// add requesrUUID to metadataMap +func AddRequestUUIDToMap(metadataMap map[string]string, requestUUID string) map[string]string{ + if len(requestUUID) > 0 { + metadataMap["RequestID"] = requestUUID + } + return metadataMap +} + +// convert metadataMap to a string, aims to support fmt.Errorf +func ConvertMapToString(metadataMap map[string]string) string { + metadata := "" + if len(metadataMap) != 0 { + for k, v := range metadataMap { + if len(k) != 0 && len(v) != 0 { + metadata = fmt.Sprintf("%s, %s: %s", metadata, k, v) + } } } - return InfoMetaDataMap + return metadata; } diff --git a/pkg/tester/construction.go b/pkg/tester/construction.go index ad26185c..9c181388 100644 --- a/pkg/tester/construction.go +++ b/pkg/tester/construction.go @@ -54,6 +54,7 @@ const ( ) var _ http.Handler = (*ConstructionTester)(nil) +var constructionMetadata string // ConstructionTester coordinates the `check:construction` test. type ConstructionTester struct { @@ -84,11 +85,18 @@ func InitializeConstruction( signalReceived *bool, ) (*ConstructionTester, error) { dataPath, err := utils.CreateCommandPath(config.DataDirectory, constructionCmdName, network) + metadataMap := logger.ConvertStringToMap(config.InfoMetaData) + metadataMap = logger.AddRequestUUIDToMap(metadataMap, config.RequestUUID) + constructionMetadata = logger.ConvertMapToString(metadataMap) if err != nil { - return nil, fmt.Errorf("failed to create command path: %w", err) + err = fmt.Errorf("failed to create command path: %w%s", err, constructionMetadata) + color.Red(err.Error()) + return nil, err } opts := []database.BadgerOption{} + // add constructionMetadata into localStore + opts = append(opts, database.WithMetaData(constructionMetadata)) if config.CompressionDisabled { opts = append(opts, database.WithoutCompression()) } @@ -99,14 +107,19 @@ func InitializeConstruction( ) } + // add constructionMetadata into localStore localStore, err := database.NewBadgerDatabase(ctx, dataPath, opts...) if err != nil { - return nil, fmt.Errorf("unable to initialize database: %w", err) + err = fmt.Errorf("unable to initialize database: %w%s", err, constructionMetadata) + color.Red(err.Error()) + return nil, err } networkOptions, fetchErr := onlineFetcher.NetworkOptionsRetry(ctx, network, nil) - if err != nil { - return nil, fmt.Errorf("unable to get network options: %w", fetchErr.Err) + if fetchErr != nil { + err := fmt.Errorf("unable to get network options: %w%s", fetchErr.Err, constructionMetadata) + color.Red(err.Error()) + return nil, err } if len(networkOptions.Allow.BalanceExemptions) > 0 && @@ -115,7 +128,7 @@ func InitializeConstruction( } counterStorage := modules.NewCounterStorage(localStore) - logInfoMetaData := logger.ConvertStringToMap(config.InfoMetaData) + //add constructionMetadata into logger logger, err := logger.NewLogger( dataPath, false, @@ -124,11 +137,12 @@ func InitializeConstruction( false, logger.Construction, network, - config.RequestUUID, - logInfoMetaData, + metadataMap, ) if err != nil { - return nil, fmt.Errorf("unable to initialize logger with error: %w", err) + err = fmt.Errorf("unable to initialize logger with error: %w%s", err, constructionMetadata) + color.Red(err.Error()) + return nil, err } blockStorage := modules.NewBlockStorage(localStore, config.SerialBlockWorkers) @@ -173,12 +187,13 @@ func InitializeConstruction( blockStorage, onlineFetcher, ) - + //add constructionMetadata into fetcher fetcherOpts := []fetcher.Option{ fetcher.WithMaxConnections(config.Construction.MaxOfflineConnections), fetcher.WithAsserter(onlineFetcher.Asserter), fetcher.WithTimeout(time.Duration(config.HTTPTimeout) * time.Second), fetcher.WithMaxRetries(config.MaxRetries), + fetcher.WithMetaData(constructionMetadata), } if config.Construction.ForceRetry { fetcherOpts = append(fetcherOpts, fetcher.WithForceRetry()) @@ -192,13 +207,17 @@ func InitializeConstruction( // Import prefunded account and save to database err = keyStorage.ImportAccounts(ctx, config.Construction.PrefundedAccounts) if err != nil { + err = fmt.Errorf("%w%s", err, constructionMetadata) + color.Red(err.Error()) return nil, err } // Load all accounts for network accounts, err := keyStorage.GetAllAccounts(ctx) if err != nil { - return nil, fmt.Errorf("unable to load addresses: %w", err) + err = fmt.Errorf("unable to load addresses: %w%s", err, constructionMetadata) + color.Red(err.Error()) + return nil, err } // Track balances on all addresses @@ -206,7 +225,7 @@ func InitializeConstruction( balanceStorageHelper.AddInterestingAddress(account.Address) } - log.Printf("construction tester initialized with %d accounts\n", len(accounts)) + color.Cyan("construction tester initialized with %d accounts%s\n", len(accounts), constructionMetadata) // Load prefunded accounts var accountBalanceRequests []*utils.AccountBalanceRequest @@ -234,12 +253,16 @@ func InitializeConstruction( accBalances, err := utils.GetAccountBalances(ctx, onlineFetcher, accountBalanceRequests) if err != nil { - return nil, fmt.Errorf("unable to get account balances: %w", err) + err = fmt.Errorf("unable to get account balances: %w%s", err, constructionMetadata) + color.Red(err.Error()) + return nil, err } err = balanceStorage.SetBalanceImported(ctx, nil, accBalances) if err != nil { - return nil, fmt.Errorf("unable to set balances: %w", err) + err = fmt.Errorf("unable to set balances: %w%s", err, constructionMetadata) + color.Red(err.Error()) + return nil, err } // ------------------------------------------------------------------------- @@ -249,7 +272,9 @@ func InitializeConstruction( if config.CoinSupported { acctCoins, errAccCoins := utils.GetAccountCoins(ctx, onlineFetcher, acctCoinsReqs) if errAccCoins != nil { - return nil, fmt.Errorf("unable to get account coins: %w", errAccCoins) + err = fmt.Errorf("unable to get account coins: %w%s", errAccCoins, constructionMetadata) + color.Red(err.Error()) + return nil, err } // Extract accounts from account coins requests @@ -260,7 +285,9 @@ func InitializeConstruction( err = coinStorage.SetCoinsImported(ctx, accts, acctCoins) if err != nil { - return nil, fmt.Errorf("unable to set coin balances: %w", err) + err = fmt.Errorf("unable to set coin balances: %w%s", err, constructionMetadata) + color.Red(err.Error()) + return nil, err } } @@ -294,7 +321,9 @@ func InitializeConstruction( config.Construction.Workflows, ) if err != nil { - log.Fatalf("unable to create coordinator: %s", err.Error()) + msg := fmt.Sprintf("unable to create coordinator: %s%s", err.Error(), constructionMetadata) + color.Red(msg) + log.Fatalf(msg) } broadcastHandler := processor.NewBroadcastStorageHandler( @@ -307,6 +336,7 @@ func InitializeConstruction( broadcastStorage.Initialize(broadcastHelper, broadcastHandler) + //add constructionMetadata into syncer syncer := statefulsyncer.New( ctx, network, @@ -320,6 +350,7 @@ func InitializeConstruction( statefulsyncer.WithMaxConcurrency(config.MaxSyncConcurrency), statefulsyncer.WithPastBlockLimit(config.MaxReorgDepth), statefulsyncer.WithSeenConcurrency(int64(config.SeenBlockWorkers)), + statefulsyncer.WithMetaData(constructionMetadata), ) return &ConstructionTester{ @@ -342,7 +373,9 @@ func InitializeConstruction( // CloseDatabase closes the database used by ConstructionTester. func (t *ConstructionTester) CloseDatabase(ctx context.Context) { if err := t.database.Close(ctx); err != nil { - log.Fatalf("error closing database: %s", err.Error()) + msg := fmt.Sprintf("error closing database: %s%s", err.Error(), constructionMetadata) + color.Red(msg) + log.Fatalf(msg) } } @@ -379,7 +412,9 @@ func (t *ConstructionTester) checkTip(ctx context.Context) (int64, error) { t.onlineFetcher, ) if err != nil { - return -1, fmt.Errorf("failed to check network tip: %w", err) + err = fmt.Errorf("failed to check network tip: %w%s", err, constructionMetadata) + color.Red(err.Error()) + return -1, err } if atTip { @@ -398,14 +433,16 @@ func (t *ConstructionTester) waitForTip(ctx context.Context) (int64, error) { // Don't wait any time before first tick if at tip. tipIndex, err := t.checkTip(ctx) if err != nil { - return -1, fmt.Errorf("failed to check tip: %w", err) + err = fmt.Errorf("failed to check tip: %w%s", err, constructionMetadata) + color.Red(err.Error()) + return -1, err } if tipIndex != -1 { return tipIndex, nil } - log.Println("waiting for implementation to reach tip before testing...") + color.Cyan("waiting for implementation to reach tip before testing...%s", constructionMetadata) select { case <-ctx.Done(): @@ -430,10 +467,14 @@ func (t *ConstructionTester) StartSyncer( // we will unnecessarily sync tons of blocks before reaching any that matter. startIndex, err = t.waitForTip(ctx) if err != nil { - return fmt.Errorf("unable to wait for tip: %w", err) + err = fmt.Errorf("unable to wait for tip: %w%s", err, constructionMetadata) + color.Red(err.Error()) + return err } } else if err != nil { - return fmt.Errorf("unable to get last block synced: %w", err) + err = fmt.Errorf("unable to get last block synced: %w%s", err, constructionMetadata) + color.Red(err.Error()) + return err } return t.syncer.Sync(ctx, startIndex, -1) @@ -448,10 +489,12 @@ func (t *ConstructionTester) StartConstructor( if t.config.Construction.ClearBroadcasts { broadcasts, err := t.broadcastStorage.ClearBroadcasts(ctx) if err != nil { - return fmt.Errorf("unable to clear broadcasts: %w", err) + err = fmt.Errorf("unable to clear broadcasts: %w%s", err, constructionMetadata) + color.Red(err.Error()) + return err } - log.Printf("cleared %d broadcasts\n", len(broadcasts)) + color.Cyan("cleared %d broadcasts%s\n", len(broadcasts), constructionMetadata) } return t.coordinator.Process(ctx) @@ -482,10 +525,12 @@ func (t *ConstructionTester) PerformBroadcasts(ctx context.Context) error { return nil } - color.Magenta("Rebroadcasting all transactions...") + color.Magenta("Rebroadcasting all transactions...%s", constructionMetadata) if err := t.broadcastStorage.BroadcastAll(ctx, false); err != nil { - return fmt.Errorf("unable to broadcast all transactions: %w", err) + err = fmt.Errorf("unable to broadcast all transactions: %w%s", err, constructionMetadata) + color.Red(err.Error()) + return err } return nil @@ -514,7 +559,9 @@ func (t *ConstructionTester) WatchEndConditions( for workflow, minOccurences := range endConditions { completed, err := t.jobStorage.Completed(ctx, workflow) if err != nil { - return fmt.Errorf("unable to fetch completed %s: %w", workflow, err) + err = fmt.Errorf("unable to fetch completed %s: %w%s", workflow, err, constructionMetadata) + color.Red(err.Error()) + return err } if len(completed) < minOccurences { @@ -564,12 +611,12 @@ func (t *ConstructionTester) returnFunds( err := g.Wait() if *t.signalReceived { - color.Red("Fund return halted") + color.Red("Fund return halted%s", constructionMetadata) return } if !returnFundsSuccess { - log.Printf("unable to return funds %v\n", err) + color.Cyan("unable to return funds %v%s\n", err, constructionMetadata) } } @@ -579,15 +626,18 @@ func (t *ConstructionTester) HandleErr( sigListeners *[]context.CancelFunc, ) error { if *t.signalReceived { + err = fmt.Errorf("%v: %w%s", err.Error(), cliErrs.ErrConstructionCheckHalt, constructionMetadata) + color.Red(err.Error()) return results.ExitConstruction( t.config, t.counterStorage, t.jobStorage, - fmt.Errorf("%v: %w", err.Error(), cliErrs.ErrConstructionCheckHalt), + err, ) } if !t.reachedEndConditions { + color.Red("%v%s", err, constructionMetadata) return results.ExitConstruction(t.config, t.counterStorage, t.jobStorage, err) } diff --git a/pkg/tester/data.go b/pkg/tester/data.go index b687e885..0487fa62 100644 --- a/pkg/tester/data.go +++ b/pkg/tester/data.go @@ -85,6 +85,7 @@ const ( var _ http.Handler = (*DataTester)(nil) var _ statefulsyncer.PruneHelper = (*DataTester)(nil) +var metadata string // DataTester coordinates the `check:data` test. type DataTester struct { @@ -131,15 +132,19 @@ func loadAccounts(filePath string) ([]*types.AccountCurrency, error) { accounts := []*types.AccountCurrency{} if err := utils.LoadAndParse(filePath, &accounts); err != nil { - return nil, fmt.Errorf("unable to load and parse %s: %w", filePath, err) + err = fmt.Errorf("unable to load and parse %s: %w%s", filePath, err, metadata) + color.Red(err.Error()) + return nil, err } - log.Printf( - "Found %d accounts at %s: %s\n", + msg := fmt.Sprintf( + "Found %d accounts at %s: %s%s\n", len(accounts), filePath, types.PrettyPrintStruct(accounts), + metadata, ) + color.Cyan(msg) return accounts, nil } @@ -147,7 +152,9 @@ func loadAccounts(filePath string) ([]*types.AccountCurrency, error) { // CloseDatabase closes the database used by DataTester. func (t *DataTester) CloseDatabase(ctx context.Context) { if err := t.database.Close(ctx); err != nil { - log.Fatalf("error closing database: %s", err.Error()) + msg := fmt.Sprintf("error closing database: %s%s", err.Error(), metadata) + color.Red(msg) + log.Fatalf(msg) } } @@ -163,11 +170,19 @@ func InitializeData( signalReceived *bool, ) (*DataTester, error) { dataPath, err := utils.CreateCommandPath(config.DataDirectory, dataCmdName, network) + metadataMap := logger.ConvertStringToMap(config.InfoMetaData) + metadataMap = logger.AddRequestUUIDToMap(metadataMap, config.RequestUUID) + metadata = logger.ConvertMapToString(metadataMap) + if err != nil { - return nil, fmt.Errorf("failed to create command path: %w", err) + err = fmt.Errorf("failed to create command path: %w%s", err, metadata) + color.Red(err.Error()) + return nil, err } + //add metadata into localStore opts := []database.BadgerOption{} + opts = append(opts, database.WithMetaData(metadata)) dataPathBackup := dataPath if config.AllInMemoryEnabled { @@ -213,26 +228,31 @@ func InitializeData( } } + //add metadata into localStore localStore, err := database.NewBadgerDatabase(ctx, dataPathBackup, opts...) if err != nil { - return nil, fmt.Errorf("unable to initialize database: %w", err) + err = fmt.Errorf("unable to initialize database: %w%s", err, metadata) + color.Red(err.Error()) + return nil, err } exemptAccounts, err := loadAccounts(config.Data.ExemptAccounts) if err != nil { - return nil, fmt.Errorf("unable to load exempt accounts: %w", err) + err = fmt.Errorf("unable to load exempt accounts: %w%s", err, metadata) + color.Red(err.Error()) + return nil, err } interestingAccounts, err := loadAccounts(config.Data.InterestingAccounts) if err != nil { - return nil, fmt.Errorf("unable to load interesting accounts: %w", err) + err = fmt.Errorf("unable to load interesting accounts: %w%s", err, metadata) + color.Red(err.Error()) + return nil, err } counterStorage := modules.NewCounterStorage(localStore) blockStorage := modules.NewBlockStorage(localStore, config.SerialBlockWorkers) balanceStorage := modules.NewBalanceStorage(localStore) - - logInfoMetaData := logger.ConvertStringToMap(config.InfoMetaData) logger, err := logger.NewLogger( dataPath, config.Data.LogBlocks, @@ -241,11 +261,12 @@ func InitializeData( config.Data.LogReconciliations, logger.Data, network, - config.RequestUUID, - logInfoMetaData, + metadataMap, ) if err != nil { - return nil, fmt.Errorf("unable to initialize logger with error: %w", err) + err = fmt.Errorf("unable to initialize logger with error: %w%s", err, metadata) + color.Red(err.Error()) + return nil, err } var forceInactiveReconciliation bool @@ -258,7 +279,6 @@ func InitializeData( balanceStorage, &forceInactiveReconciliation, ) - reconcilerHandler := processor.NewReconcilerHandler( logger, counterStorage, @@ -269,16 +289,22 @@ func InitializeData( // Get all previously seen accounts seenAccounts, err := balanceStorage.GetAllAccountCurrency(ctx) if err != nil { - return nil, fmt.Errorf("unable to get previously seen accounts: %w", err) + err = fmt.Errorf("unable to get previously seen accounts: %w%s", err, metadata) + color.Red(err.Error()) + return nil, err } networkOptions, fetchErr := fetcher.NetworkOptionsRetry(ctx, network, nil) if fetchErr != nil { - log.Fatalf("unable to get network options: %s", fetchErr.Err.Error()) + msg := fmt.Sprintf("unable to get network options: %s%s", fetchErr.Err.Error(), metadata) + color.Red(msg) + log.Fatalf(msg) } if len(networkOptions.Allow.BalanceExemptions) > 0 && config.Data.InitialBalanceFetchDisabled { - return nil, fmt.Errorf("found balance exemptions but initial balance fetch disabled") + err = fmt.Errorf("found balance exemptions but initial balance fetch disabled%s", metadata) + color.Red(err.Error()) + return nil, err } parser := parser.New( @@ -295,6 +321,7 @@ func InitializeData( historicalBalanceEnabled = networkOptions.Allow.HistoricalBalanceLookup } + //add metadata into reconciler rOpts := []reconciler.Option{ reconciler.WithActiveConcurrency(int(config.Data.ActiveReconciliationConcurrency)), reconciler.WithInactiveConcurrency(int(config.Data.InactiveReconciliationConcurrency)), @@ -302,6 +329,7 @@ func InitializeData( reconciler.WithSeenAccounts(seenAccounts), reconciler.WithInactiveFrequency(int64(config.Data.InactiveReconciliationFrequency)), reconciler.WithBalancePruning(), + reconciler.WithMetaData(metadata), } if config.Data.ReconcilerActiveBacklog != nil { rOpts = append(rOpts, reconciler.WithBacklogSize(*config.Data.ReconcilerActiveBacklog)) @@ -313,6 +341,7 @@ func InitializeData( rOpts = append(rOpts, reconciler.WithDebugLogging()) } + //add metadata into reconciler r := reconciler.New( reconcilerHelper, reconcilerHandler, @@ -366,12 +395,16 @@ func InitializeData( genesisBlock, ) if err != nil { - return nil, fmt.Errorf("unable to bootstrap balances: %w", err) + err = fmt.Errorf("unable to bootstrap balances: %w%s", err, metadata) + color.Red(err.Error()) + return nil, err } case err != nil: - return nil, fmt.Errorf("unable to get head block identifier: %w", err) + err = fmt.Errorf("unable to get head block identifier: %w%s", err, metadata) + color.Red(err.Error()) + return nil, err default: - log.Println("Skipping balance bootstrapping because already started syncing") + color.Cyan("Skipping balance bootstrapping because already started syncing%s", metadata) } } } @@ -383,11 +416,13 @@ func InitializeData( blockWorkers = append(blockWorkers, coinStorage) } + //add metadata into statefulsyncer statefulSyncerOptions := []statefulsyncer.Option{ statefulsyncer.WithCacheSize(syncer.DefaultCacheSize), statefulsyncer.WithMaxConcurrency(config.MaxSyncConcurrency), statefulsyncer.WithPastBlockLimit(config.MaxReorgDepth), statefulsyncer.WithSeenConcurrency(int64(config.SeenBlockWorkers)), + statefulsyncer.WithMetaData(metadata), } if config.Data.PruningFrequency != nil { statefulSyncerOptions = append( @@ -396,6 +431,7 @@ func InitializeData( ) } + //add metadata into syncer syncer := statefulsyncer.New( ctx, network, @@ -545,7 +581,10 @@ func (t *DataTester) ServeHTTP(w http.ResponseWriter, r *http.Request) { ) if err := json.NewEncoder(w).Encode(status); err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) + msg := err.Error() + msg = fmt.Sprintf("%s%s", msg, metadata) + color.Red(msg) + http.Error(w, msg, http.StatusInternalServerError) } } @@ -560,7 +599,9 @@ func (t *DataTester) syncedStatus(ctx context.Context) (bool, int64, error) { t.blockStorage, ) if err != nil { - return false, -1, fmt.Errorf("failed to check storage tip: %w", err) + err = fmt.Errorf("failed to check storage tip: %w%s", err, metadata) + color.Red(err.Error()) + return false, -1, err } var blockIndex int64 = -1 @@ -587,9 +628,10 @@ func (t *DataTester) EndAtTipLoop( case <-tc.C: atTip, blockIndex, err := t.syncedStatus(ctx) if err != nil { - log.Printf( - "unable to evaluate if syncer is at tip: %s", + color.Red( + "unable to evaluate if syncer is at tip: %s%s", err.Error(), + metadata, ) continue } @@ -600,6 +642,12 @@ func (t *DataTester) EndAtTipLoop( "Tip: %d", blockIndex, ) + msg := fmt.Sprintf( + "%s%s", + t.endConditionDetail, + metadata, + ) + color.Cyan(msg) t.cancel() return } @@ -625,9 +673,10 @@ func (t *DataTester) EndReconciliationCoverage( // nolint:gocognit case <-tc.C: atTip, blockIndex, err := t.syncedStatus(ctx) if err != nil { - log.Printf( - "unable to evaluate syncer height or if at tip: %s", + color.Red( + "unable to evaluate syncer height or if at tip: %s%s", err.Error(), + metadata, ) continue } @@ -686,9 +735,10 @@ func (t *DataTester) EndReconciliationCoverage( // nolint:gocognit if reconciliationCoverage.AccountCount != nil { allAccounts, err := t.balanceStorage.GetAllAccountCurrency(ctx) if err != nil { - log.Printf( - "unable to get account count: %s", + color.Red( + "unable to get account count: %s%s", err.Error(), + metadata, ) continue } @@ -705,9 +755,10 @@ func (t *DataTester) EndReconciliationCoverage( // nolint:gocognit coverage, err := t.balanceStorage.ReconciliationCoverage(ctx, coverageIndex) if err != nil { - log.Printf( - "unable to get reconciliation coverage: %s", + color.Red( + "unable to get reconciliation coverage: %s%s", err.Error(), + metadata, ) continue } @@ -715,18 +766,25 @@ func (t *DataTester) EndReconciliationCoverage( // nolint:gocognit if coverage >= reconciliationCoverage.Coverage { t.endCondition = configuration.ReconciliationCoverageEndCondition t.endConditionDetail = fmt.Sprintf( - "Coverage: %f%%", + "Coverage: %f", coverage*utils.OneHundred, ) + msg := fmt.Sprintf( + "%s%s", + t.endConditionDetail, + metadata, + ) + color.Cyan(msg) t.cancel() return } color.Cyan(fmt.Sprintf( - "[END CONDITIONS] Waiting for reconciliation coverage after block %d (%f%%) to surpass requirement (%f%%)", + "[END CONDITIONS] Waiting for reconciliation coverage after block %d (%f%%) to surpass requirement (%f%%)%s", firstTipIndex, coverage*utils.OneHundred, reconciliationCoverage.Coverage*utils.OneHundred, + metadata, )) } } @@ -751,6 +809,12 @@ func (t *DataTester) EndDurationLoop( "Seconds: %d", int(duration.Seconds()), ) + msg := fmt.Sprintf( + "%s%s", + t.endConditionDetail, + metadata, + ) + color.Cyan(msg) t.cancel() return } @@ -788,22 +852,30 @@ func (t *DataTester) WatchEndConditions( func (t *DataTester) CompleteReconciliations(ctx context.Context) (int64, error) { activeReconciliations, err := t.counterStorage.Get(ctx, modules.ActiveReconciliationCounter) if err != nil { - return -1, fmt.Errorf("failed to get active reconciliations counter: %w", err) + err = fmt.Errorf("failed to get active reconciliations counter: %w%s", err, metadata) + color.Red(err.Error()) + return -1, fmt.Errorf("failed to get active reconciliations counter: %w%s", err, metadata) } exemptReconciliations, err := t.counterStorage.Get(ctx, modules.ExemptReconciliationCounter) if err != nil { - return -1, fmt.Errorf("failed to get exempt reconciliations counter: %w", err) + err = fmt.Errorf("failed to get exempt reconciliations counter: %w%s", err, metadata) + color.Red(err.Error()) + return -1, err } failedReconciliations, err := t.counterStorage.Get(ctx, modules.FailedReconciliationCounter) if err != nil { - return -1, fmt.Errorf("failed to get failed reconciliations counter: %w", err) + err = fmt.Errorf("failed to get failed reconciliations counter: %w%s", err, metadata) + color.Red(err.Error()) + return -1, err } skippedReconciliations, err := t.counterStorage.Get(ctx, modules.SkippedReconciliationsCounter) if err != nil { - return -1, fmt.Errorf("failed to get skipped reconciliations counter: %w", err) + err = fmt.Errorf("failed to get skipped reconciliations counter: %w%s", err, metadata) + color.Red(err.Error()) + return -1, err } return activeReconciliations.Int64() + @@ -823,7 +895,9 @@ func (t *DataTester) WaitForEmptyQueue( // and only exit when that many reconciliations have been performed. startingComplete, err := t.CompleteReconciliations(ctx) if err != nil { - return fmt.Errorf("failed to complete reconciliations: %w", err) + err = fmt.Errorf("failed to complete reconciliations: %w%s", err, metadata) + color.Red(err.Error()) + return err } startingRemaining := t.reconciler.QueueSize() @@ -831,8 +905,9 @@ func (t *DataTester) WaitForEmptyQueue( defer tc.Stop() color.Cyan( - "[PROGRESS] remaining reconciliations: %d", + "[PROGRESS] remaining reconciliations: %d%s", startingRemaining, + metadata, ) for { @@ -844,12 +919,16 @@ func (t *DataTester) WaitForEmptyQueue( // We force cached counts to be written before // determining if we should exit. if err := t.reconcilerHandler.UpdateCounts(ctx); err != nil { - return fmt.Errorf("failed to update count: %w", err) + err = fmt.Errorf("failed to update count: %w%s", err, metadata) + color.Red(err.Error()) + return err } nowComplete, err := t.CompleteReconciliations(ctx) if err != nil { - return fmt.Errorf("failed to complete reconciliations: %w", err) + err = fmt.Errorf("failed to complete reconciliations: %w%s", err, metadata) + color.Red(err.Error()) + return err } completed := nowComplete - startingComplete @@ -860,8 +939,9 @@ func (t *DataTester) WaitForEmptyQueue( } color.Cyan( - "[PROGRESS] remaining reconciliations: %d", + "[PROGRESS] remaining reconciliations: %d%s", remaining, + metadata, ) } } @@ -873,7 +953,7 @@ func (t *DataTester) DrainReconcilerQueue( ctx context.Context, sigListeners *[]context.CancelFunc, ) error { - color.Cyan("draining reconciler backlog (you can disable this in your configuration file)") + color.Cyan("draining reconciler backlog (you can disable this in your configuration file)%s", metadata) // To cancel all execution, need to call multiple cancel functions. ctx, cancel := context.WithCancel(ctx) @@ -898,7 +978,7 @@ func (t *DataTester) DrainReconcilerQueue( } if errors.Is(err, context.Canceled) { - color.Cyan("drained reconciler backlog") + color.Cyan("drained reconciler backlog%s", metadata) return nil } @@ -914,11 +994,13 @@ func (t *DataTester) HandleErr(err error, sigListeners *[]context.CancelFunc) er ctx := context.Background() if *t.signalReceived { + err = fmt.Errorf("%v: %w%s", err.Error(), cliErrs.ErrDataCheckHalt, metadata) + color.Red(err.Error()) return results.ExitData( t.config, t.counterStorage, t.balanceStorage, - fmt.Errorf("%v: %w", err.Error(), cliErrs.ErrDataCheckHalt), + err, "", "", ) @@ -932,6 +1014,12 @@ func (t *DataTester) HandleErr(err error, sigListeners *[]context.CancelFunc) er "Index: %d", *t.config.Data.EndConditions.Index, ) + msg := fmt.Sprintf( + "%s%s", + t.endConditionDetail, + metadata, + ) + color.Cyan(msg) } // End condition will only be populated if there is @@ -942,16 +1030,19 @@ func (t *DataTester) HandleErr(err error, sigListeners *[]context.CancelFunc) er t.reconciler.QueueSize() > 0 { if t.config.Data.ReconciliationDrainDisabled { color.Cyan( - "skipping reconciler backlog drain (you can enable this in your configuration file)", + "skipping reconciler backlog drain (you can enable this in your configuration file)%s", + metadata, ) } else { drainErr := t.DrainReconcilerQueue(ctx, sigListeners) if drainErr != nil { + err = fmt.Errorf("%w%s", drainErr, metadata) + color.Red(err.Error()) return results.ExitData( t.config, t.counterStorage, t.balanceStorage, - drainErr, + err, "", "", ) @@ -971,6 +1062,8 @@ func (t *DataTester) HandleErr(err error, sigListeners *[]context.CancelFunc) er fmt.Printf("\n") if t.reconcilerHandler.InactiveFailure == nil { + err = fmt.Errorf("%w%s", err, metadata) + color.Red(err.Error()) return results.ExitData( t.config, t.counterStorage, @@ -983,8 +1076,11 @@ func (t *DataTester) HandleErr(err error, sigListeners *[]context.CancelFunc) er if !t.historicalBalanceEnabled { color.Yellow( - "Can't find the block missing operations automatically, please enable historical balance lookup", + "Can't find the block missing operations automatically, please enable historical balance lookup%s", + metadata, ) + err = fmt.Errorf("%w%s", err, metadata) + color.Red(err.Error()) return results.ExitData( t.config, t.counterStorage, @@ -996,7 +1092,9 @@ func (t *DataTester) HandleErr(err error, sigListeners *[]context.CancelFunc) er } if t.config.Data.InactiveDiscrepancySearchDisabled { - color.Yellow("Search for inactive reconciliation discrepancy is disabled") + color.Yellow("Search for inactive reconciliation discrepancy is disabled%s", metadata) + err = fmt.Errorf("%w%s", err, metadata) + color.Red(err.Error()) return results.ExitData( t.config, t.counterStorage, @@ -1007,7 +1105,11 @@ func (t *DataTester) HandleErr(err error, sigListeners *[]context.CancelFunc) er ) } - return t.FindMissingOps(ctx, err, sigListeners) + return t.FindMissingOps( + ctx, + err, + sigListeners, + ) } // FindMissingOps logs the types.BlockIdentifier of a block @@ -1018,7 +1120,7 @@ func (t *DataTester) FindMissingOps( originalErr error, sigListeners *[]context.CancelFunc, ) error { - color.Cyan("Searching for block with missing operations...hold tight") + color.Cyan("Searching for block with missing operations...hold tight%s", metadata) badBlock, err := t.recursiveOpSearch( ctx, sigListeners, @@ -1027,7 +1129,7 @@ func (t *DataTester) FindMissingOps( t.reconcilerHandler.InactiveFailureBlock.Index, ) if err != nil { - color.Yellow("could not find block with missing ops: %s", err.Error()) + color.Yellow("could not find block with missing ops: %s%s", err.Error(), metadata) return results.ExitData( t.config, t.counterStorage, @@ -1039,10 +1141,11 @@ func (t *DataTester) FindMissingOps( } color.Yellow( - "Missing ops for %s in block %d:%s", + "Missing ops for %s in block %d:%s%s", types.AccountString(t.reconcilerHandler.InactiveFailure.Account), badBlock.Index, badBlock.Hash, + metadata, ) return results.ExitData( @@ -1069,13 +1172,17 @@ func (t *DataTester) recursiveOpSearch( // Always use a temporary directory to find missing ops tmpDir, err := utils.CreateTempDir() if err != nil { - return nil, fmt.Errorf("unable to create temporary directory: %w", err) + err = fmt.Errorf("unable to create temporary directory: %w%s", err, metadata) + color.Red(err.Error()) + return nil, err } defer utils.RemoveTempDir(tmpDir) localStore, err := database.NewBadgerDatabase(ctx, tmpDir) if err != nil { - return nil, fmt.Errorf("unable to initialize database: %w", err) + err = fmt.Errorf("unable to initialize database: %w%s", err, metadata) + color.Red(err.Error()) + return nil, err } counterStorage := modules.NewCounterStorage(localStore) @@ -1090,12 +1197,13 @@ func (t *DataTester) recursiveOpSearch( false, logger.Data, t.network, - EmptyRequestUUID, - nil, + t.logger.GetMetadataMap(), ) if err != nil { - return nil, fmt.Errorf("unable to initialize logger with error: %w", err) + err = fmt.Errorf("unable to initialize logger with error: %w%s", err, metadata) + color.Red(err.Error()) + return nil, err } t.forceInactiveReconciliation = types.Bool(false) @@ -1187,7 +1295,9 @@ func (t *DataTester) recursiveOpSearch( // Close database before starting another search, otherwise we will // have n databases open when we find the offending block. if storageErr := localStore.Close(ctx); storageErr != nil { - return nil, fmt.Errorf("unable to close database: %w", storageErr) + err = fmt.Errorf("unable to close database: %w%s", storageErr, metadata) + color.Red(err.Error()) + return nil, err } if *t.signalReceived { @@ -1206,18 +1316,22 @@ func (t *DataTester) recursiveOpSearch( newEnd := endIndex - InactiveFailureLookbackWindow if newEnd <= newStart { - return nil, fmt.Errorf( - "next window to check has start index %d <= end index %d", + err = fmt.Errorf( + "next window to check has start index %d <= end index %d%s", newStart, newEnd, + metadata, ) + color.Red(err.Error()) + return nil, err } color.Cyan( - "Unable to find missing ops in block range %d-%d, now searching %d-%d", + "Unable to find missing ops in block range %d-%d, now searching %d-%d%s", startIndex, endIndex, newStart, newEnd, + metadata, ) return t.recursiveOpSearch( From 87f7b061a433c158ab7b12f54f6dbc21e1456e8e Mon Sep 17 00:00:00 2001 From: jingweicb <96205264+jingweicb@users.noreply.github.com> Date: Wed, 18 Jan 2023 11:08:18 -0800 Subject: [PATCH 371/376] bump up to 0.10.2 (#383) --- Dockerfile | 2 +- cmd/root.go | 2 +- pkg/processor/balance_storage_helper.go | 14 +++++++++++++- 3 files changed, 15 insertions(+), 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index 1545c72b..07d7bbdc 100644 --- a/Dockerfile +++ b/Dockerfile @@ -31,7 +31,7 @@ RUN mkdir -p "$GOPATH/src" "$GOPATH/bin" && chmod -R 777 "$GOPATH" WORKDIR /go/src -ARG VERSION=v0.10.1 +ARG VERSION=v0.10.2 RUN git clone https://github.com/coinbase/rosetta-cli.git && \ cd rosetta-cli && \ git fetch --all --tags && \ diff --git a/cmd/root.go b/cmd/root.go index 747239ac..3cca1e7b 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -488,6 +488,6 @@ var versionCmd = &cobra.Command{ Use: "version", Short: "Print rosetta-cli version", Run: func(cmd *cobra.Command, args []string) { - fmt.Println("v0.10.1") + fmt.Println("v0.10.2") }, } diff --git a/pkg/processor/balance_storage_helper.go b/pkg/processor/balance_storage_helper.go index f043691d..6f929b47 100644 --- a/pkg/processor/balance_storage_helper.go +++ b/pkg/processor/balance_storage_helper.go @@ -65,7 +65,13 @@ func NewBalanceStorageHelper( // Pre-process exemptAccounts on initialization // to provide fast lookup while syncing. for _, account := range exemptAccounts { - exemptMap[types.Hash(account)] = struct{}{} + // if users do not specify Currency, we add the address + // by this, all the Currencies in this address will be skipped + if account.Currency == nil { + exemptMap[account.Account.Address] = struct{}{} + } else { + exemptMap[types.Hash(account)] = struct{}{} + } } return &BalanceStorageHelper{ @@ -144,6 +150,12 @@ func (h *BalanceStorageHelper) ExemptFunc() parser.ExemptOperation { return true } } + // if exemptAccounts have the Account address means all the + // currencies in this Account address need to be skipped + _, existsAddress := h.exemptAccounts[op.Account.Address] + if existsAddress { + return existsAddress + } thisAcct := types.Hash(&types.AccountCurrency{ Account: op.Account, From 1188c48cef9717173b418065442e54599b2ee7ad Mon Sep 17 00:00:00 2001 From: Madhur Shrimal Date: Fri, 27 Jan 2023 14:54:53 -0800 Subject: [PATCH 372/376] add sign tool (#386) * add sign tool * add ex * make fmt * log error --- cmd/check_data.go | 4 +- cmd/root.go | 5 +- cmd/sign.go | 72 ++++++++++++++++++++++++ cmd/utils_asserter_configuration_test.go | 24 ++++---- cmd/validate_asserter_config_test.go | 16 +++--- configuration/types.go | 14 +++-- examples/configuration/default.json | 3 +- examples/configuration/sign.json | 13 +++++ pkg/logger/logger.go | 16 +++--- pkg/processor/balance_storage_helper.go | 6 +- pkg/tester/data.go | 4 +- 11 files changed, 136 insertions(+), 41 deletions(-) create mode 100644 cmd/sign.go create mode 100644 examples/configuration/sign.json diff --git a/cmd/check_data.go b/cmd/check_data.go index 026285ea..535d6c71 100644 --- a/cmd/check_data.go +++ b/cmd/check_data.go @@ -25,7 +25,7 @@ import ( "github.com/coinbase/rosetta-sdk-go/fetcher" "github.com/coinbase/rosetta-sdk-go/types" "github.com/coinbase/rosetta-sdk-go/utils" - "github.com/fatih/color" + "github.com/fatih/color" "github.com/spf13/cobra" "golang.org/x/sync/errgroup" ) @@ -171,7 +171,7 @@ func runCheckDataCmd(_ *cobra.Command, _ []string) error { g, ctx := errgroup.WithContext(ctx) ctx = logger.AddMetadataMapToContext(ctx, metadataMap) - + g.Go(func() error { return dataTester.StartPeriodicLogger(ctx) }) diff --git a/cmd/root.go b/cmd/root.go index 3cca1e7b..92bb8b2e 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -375,6 +375,9 @@ default values.`, `Verify both minimum and Coinbase spec requirements`, ) rootCmd.AddCommand(checkSpecCmd) + + // Sign command + rootCmd.AddCommand(signCmd) } func initConfig() { @@ -451,7 +454,7 @@ func initConfig() { } if len(InfoMetaData) != 0 { - Config.InfoMetaData = InfoMetaData + Config.InfoMetaData = InfoMetaData } } diff --git a/cmd/sign.go b/cmd/sign.go new file mode 100644 index 00000000..4bb7cd85 --- /dev/null +++ b/cmd/sign.go @@ -0,0 +1,72 @@ +// Copyright 2023 Coinbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package cmd + +import ( + "encoding/hex" + "errors" + "fmt" + + "github.com/coinbase/rosetta-sdk-go/keys" + "github.com/fatih/color" + "github.com/spf13/cobra" +) + +var ( + signCmd = &cobra.Command{ + Use: "sign", + Short: "Sign an unsigned payload with given private key", + Long: `Sign an unsigned payload with given private key + It supports Keypair specified by https://github.com/coinbase/rosetta-specifications`, + RunE: runSignCmd, + } +) + +func runSignCmd(_ *cobra.Command, _ []string) error { + if Config.Sign == nil { + return errors.New("sign configuration is missing") + } + + keyPair, err := keys.ImportPrivateKey(Config.Sign.PrivateKey, Config.Sign.PubKey.CurveType) + if err != nil { + fmt.Println(fmt.Errorf("unable to import private keys %#v", err)) + return err + } + + err = keyPair.IsValid() + if err != nil { + fmt.Println(fmt.Errorf("keypair invalid with err %#v", err)) + return err + } + + signer, err := keyPair.Signer() + if err != nil { + fmt.Println(fmt.Errorf("signer invalid with err %#v", err)) + return err + } + + signingPayload := Config.Sign.SigningPayload + signatureType := Config.Sign.SigningPayload.SignatureType + + sign, err := signer.Sign(signingPayload, signatureType) + if err != nil { + fmt.Println(fmt.Errorf("unable to sign with err %#v", err)) + return err + } + + hexSig := hex.EncodeToString(sign.Bytes) + color.Blue(hexSig) + return nil +} diff --git a/cmd/utils_asserter_configuration_test.go b/cmd/utils_asserter_configuration_test.go index ce6f3cab..d7844e90 100644 --- a/cmd/utils_asserter_configuration_test.go +++ b/cmd/utils_asserter_configuration_test.go @@ -50,18 +50,18 @@ var ( allowedErrors = []*types.Error{ { - Code: 4, - Message: "Block not found", + Code: 4, + Message: "Block not found", Retriable: false, }, { - Code: 0, - Message: "Endpoint not implemented", + Code: 0, + Message: "Endpoint not implemented", Retriable: false, }, { - Code: 3, - Message: "Bitcoind error", + Code: 3, + Message: "Bitcoind error", Retriable: false, }, } @@ -93,18 +93,18 @@ func TestSortArrayFields(t *testing.T) { }, clientConfiguration.AllowedOperationStatuses) assert.Equal([]*types.Error{ { - Code: 0, - Message: "Endpoint not implemented", + Code: 0, + Message: "Endpoint not implemented", Retriable: false, }, { - Code: 3, - Message: "Bitcoind error", + Code: 3, + Message: "Bitcoind error", Retriable: false, }, { - Code: 4, - Message: "Block not found", + Code: 4, + Message: "Block not found", Retriable: false, }, }, clientConfiguration.AllowedErrors) diff --git a/cmd/validate_asserter_config_test.go b/cmd/validate_asserter_config_test.go index e48bf9c8..2a646056 100644 --- a/cmd/validate_asserter_config_test.go +++ b/cmd/validate_asserter_config_test.go @@ -17,8 +17,8 @@ package cmd import ( "github.com/coinbase/rosetta-sdk-go/asserter" "github.com/coinbase/rosetta-sdk-go/types" - "testing" "github.com/stretchr/testify/assert" + "testing" ) func TestMatch(t *testing.T) { @@ -104,10 +104,10 @@ func generateNetworkAllowAndAsserterConfiguration() ( ) { var tsi int64 = 5 allow := &types.Allow{ - OperationStatuses: generateOperationStatuses(), - OperationTypes: generateOperationTypes(), - Errors: generateErrors(), - TimestampStartIndex: &tsi, + OperationStatuses: generateOperationStatuses(), + OperationTypes: generateOperationTypes(), + Errors: generateErrors(), + TimestampStartIndex: &tsi, } config := &asserter.Configuration{ AllowedOperationStatuses: generateOperationStatuses(), @@ -127,7 +127,7 @@ func generateOperationStatuses() []*types.OperationStatus { return []*types.OperationStatus{ { Successful: true, - Status: "status0", + Status: "status0", }, { // Successful: false @@ -139,11 +139,11 @@ func generateOperationStatuses() []*types.OperationStatus { func generateErrors() []*types.Error { return []*types.Error{ { - Code: 1, + Code: 1, Message: "message1", }, { - Code: 2, + Code: 2, Message: "message2", }, } diff --git a/configuration/types.go b/configuration/types.go index 01763a04..8ad3a8f7 100644 --- a/configuration/types.go +++ b/configuration/types.go @@ -464,7 +464,7 @@ type Configuration struct { // InfoMetaData is a string, rosetta-cli will convert it into a map[string]string // key-value are separated by ":" // different key-value pairs are separated by "," - // an example: if users want to record "instance_name" as "1234", and "blockchain_name" as "Bitcoin", + // an example: if users want to record "instance_name" as "1234", and "blockchain_name" as "Bitcoin", // this field would be "instance_name:1234,blockchain_name:Bitcoin" // if adding spaces before and after ":" and ",", it will be trimmed when building map // " instance_name : xxxx , blockchain_name : xxxx " will be recorded same as @@ -474,11 +474,17 @@ type Configuration struct { Construction *ConstructionConfiguration `json:"construction"` Data *DataConfiguration `json:"data"` Perf *CheckPerfConfiguration `json:"perf"` + Sign *SignConfiguration `json:"sign"` } -//********************// -// Check Perf configs // -//********************// +// SignConfiguration configuration for signing +type SignConfiguration struct { + PubKey *types.PublicKey `json:"pub_key"` + PrivateKey string `json:"private_key"` + SigningPayload *types.SigningPayload `json:"signing_payload"` +} + +// CheckPerfConfiguration configuration for check perf type CheckPerfConfiguration struct { // StartBlock is the starting block for running check:perf. diff --git a/examples/configuration/default.json b/examples/configuration/default.json index d2c0c4fc..97e38a82 100644 --- a/examples/configuration/default.json +++ b/examples/configuration/default.json @@ -42,5 +42,6 @@ "pruning_balance_disabled": false, "initial_balance_fetch_disabled": false }, - "perf": null + "perf": null, + "sign": null } \ No newline at end of file diff --git a/examples/configuration/sign.json b/examples/configuration/sign.json new file mode 100644 index 00000000..c79d1f87 --- /dev/null +++ b/examples/configuration/sign.json @@ -0,0 +1,13 @@ +{ + "sign": { + "pub_key": { + "curve_type": "secp256k1", + "hex_bytes": "03c7e625aa08cad8f257d9ee2b9b7a0214f19f981afd5b498c728ad7ed6c0c3df6" + }, + "private_key": "", + "signing_payload": { + "hex_bytes": "370e74254e8cbaa343af3564901456082ec7af967e45ff24ba061233b1a1b04f", + "signature_type": "ecdsa" + } + } +} \ No newline at end of file diff --git a/pkg/logger/logger.go b/pkg/logger/logger.go index 5ade4f34..f18d34fe 100644 --- a/pkg/logger/logger.go +++ b/pkg/logger/logger.go @@ -102,7 +102,7 @@ func NewLogger( logReconciliation bool, checkType CheckType, network *types.NetworkIdentifier, - logMetadataMap map[string]string, + logMetadataMap map[string]string, fields ...zap.Field, ) (*Logger, error) { zapLogger, err := buildZapLogger(checkType, network, fields...) @@ -217,7 +217,7 @@ func (l *Logger) LogConstructionStatus( } statsMessage = AddMetadata(statsMessage, l.logMetadataMap) - + l.lastStatsMessage = statsMessage color.Cyan(statsMessage) } @@ -487,7 +487,7 @@ func (l *Logger) ReconcileSuccessStream( ) reconciliationSuccessString = AddMetadata(reconciliationSuccessString, l.logMetadataMap) color.Cyan(reconciliationSuccessString) - + _, err = f.WriteString(reconciliationSuccessString) if err != nil { err = fmt.Errorf("failed to write reconciliation success string %s: %w", reconciliationSuccessString, err) @@ -601,14 +601,14 @@ func (l *Logger) Fatal(msg string, fields ...zap.Field) { } // return a string of metadata -func (l *Logger) GetMetadata() string{ +func (l *Logger) GetMetadata() string { metadatMap := l.logMetadataMap metadata := ConvertMapToString(metadatMap) return metadata } // return a map of metadatMap -func (l *Logger) GetMetadataMap() map[string]string{ +func (l *Logger) GetMetadataMap() map[string]string { metadatMap := l.logMetadataMap return metadatMap } @@ -683,7 +683,7 @@ func ConvertStringToMap(metadata string) map[string]string { pairs := strings.Split(metadata, ",") for _, pair := range pairs { kv := strings.Split(pair, ":") - if(len(kv) != 2) { + if len(kv) != 2 { log := fmt.Sprintf("the %s from %s could be transfer to key value pair", pair, metadata) color.Yellow(log) } else { @@ -694,7 +694,7 @@ func ConvertStringToMap(metadata string) map[string]string { } // add requesrUUID to metadataMap -func AddRequestUUIDToMap(metadataMap map[string]string, requestUUID string) map[string]string{ +func AddRequestUUIDToMap(metadataMap map[string]string, requestUUID string) map[string]string { if len(requestUUID) > 0 { metadataMap["RequestID"] = requestUUID } @@ -711,5 +711,5 @@ func ConvertMapToString(metadataMap map[string]string) string { } } } - return metadata; + return metadata } diff --git a/pkg/processor/balance_storage_helper.go b/pkg/processor/balance_storage_helper.go index 6f929b47..704ec6a9 100644 --- a/pkg/processor/balance_storage_helper.go +++ b/pkg/processor/balance_storage_helper.go @@ -65,7 +65,7 @@ func NewBalanceStorageHelper( // Pre-process exemptAccounts on initialization // to provide fast lookup while syncing. for _, account := range exemptAccounts { - // if users do not specify Currency, we add the address + // if users do not specify Currency, we add the address // by this, all the Currencies in this address will be skipped if account.Currency == nil { exemptMap[account.Account.Address] = struct{}{} @@ -150,12 +150,12 @@ func (h *BalanceStorageHelper) ExemptFunc() parser.ExemptOperation { return true } } - // if exemptAccounts have the Account address means all the + // if exemptAccounts have the Account address means all the // currencies in this Account address need to be skipped _, existsAddress := h.exemptAccounts[op.Account.Address] if existsAddress { return existsAddress - } + } thisAcct := types.Hash(&types.AccountCurrency{ Account: op.Account, diff --git a/pkg/tester/data.go b/pkg/tester/data.go index 0487fa62..68afd896 100644 --- a/pkg/tester/data.go +++ b/pkg/tester/data.go @@ -78,7 +78,7 @@ const ( //MaxTableSize unit is MB MaxValueLogFileSize = int64(2048) - + // empty requestUUID EmptyRequestUUID = "" ) @@ -1106,7 +1106,7 @@ func (t *DataTester) HandleErr(err error, sigListeners *[]context.CancelFunc) er } return t.FindMissingOps( - ctx, + ctx, err, sigListeners, ) From 679464c2b49abe4161e52ef8550f4865314c76d3 Mon Sep 17 00:00:00 2001 From: Madhur Shrimal Date: Mon, 30 Jan 2023 12:35:25 -0800 Subject: [PATCH 373/376] Add signature verification tool in rosetta-cli (#387) * add verify and readme * format * verify json example --- README.md | 3 ++ cmd/README.md | 33 ++++++++++++++ cmd/{sign.go => key_sign.go} | 29 ++++++++----- cmd/key_verify.go | 69 ++++++++++++++++++++++++++++++ cmd/root.go | 7 ++- configuration/types.go | 1 + examples/configuration/sign.json | 5 ++- examples/configuration/verify.json | 18 ++++++++ 8 files changed, 151 insertions(+), 14 deletions(-) create mode 100644 cmd/README.md rename cmd/{sign.go => key_sign.go} (69%) create mode 100644 cmd/key_verify.go create mode 100644 examples/configuration/verify.json diff --git a/README.md b/README.md index 3451937d..01c2d69c 100644 --- a/README.md +++ b/README.md @@ -46,6 +46,9 @@ Example: To validate that the Data API implementation is correct, running the fo docker run -v "$(pwd):/data" -it [image-name] check:data --configuration-file /data/config.json ``` +## Key Sign Tool +Rosetta CLI comes with a handy key sign tool for local testing. Please refer to this [README](./cmd/README.md) on how to use it. + ## Updates and Releases We recommend that you continually update your installation to the latest release as soon as possible. The latest release notes are available in our [Community](https://community.rosetta-api.org) board under the [Release](https://community.rosetta-api.org/c/releases/13) category. diff --git a/cmd/README.md b/cmd/README.md new file mode 100644 index 00000000..01297daf --- /dev/null +++ b/cmd/README.md @@ -0,0 +1,33 @@ +## Key Sign Tool + +Rosetta CLI has a key sign tool, which you can use to sign and verify various curves supported +by rosetta-specifications. This should only be used for local development. Never share private keys anywhere. + +### Usage +#### Sign +``` +rosetta-cli key:sign --configuration-file config.json +``` + +A sample config file is located [here](../examples/configuration/sign.json) + +Required fields includes +- `pub_key` +- `private_key` +- `signing_payload` + + +#### Verify +``` +rosetta-cli key:verify --configuration-file verify.json +``` +A sample config file is located [here](../examples/configuration/verify.json) + +Required fields includes +- `pub_key` +- `signing_payload` +- `signature` + + +### Troubleshoot +- `account_identifier` field in `signing_payload` field should've a dummy address for providing valid payload. \ No newline at end of file diff --git a/cmd/sign.go b/cmd/key_sign.go similarity index 69% rename from cmd/sign.go rename to cmd/key_sign.go index 4bb7cd85..41a114fa 100644 --- a/cmd/sign.go +++ b/cmd/key_sign.go @@ -17,7 +17,6 @@ package cmd import ( "encoding/hex" "errors" - "fmt" "github.com/coinbase/rosetta-sdk-go/keys" "github.com/fatih/color" @@ -25,35 +24,43 @@ import ( ) var ( - signCmd = &cobra.Command{ - Use: "sign", + keySignCmd = &cobra.Command{ + Use: "key:sign", Short: "Sign an unsigned payload with given private key", Long: `Sign an unsigned payload with given private key - It supports Keypair specified by https://github.com/coinbase/rosetta-specifications`, - RunE: runSignCmd, + It supports Keypair specified by https://github.com/coinbase/rosetta-specifications + Please provide valid PrivateKey, CurveType, SignaturePayload`, + RunE: runKeySignCmd, } ) -func runSignCmd(_ *cobra.Command, _ []string) error { +func runKeySignCmd(_ *cobra.Command, _ []string) error { if Config.Sign == nil { return errors.New("sign configuration is missing") } + if len(Config.Sign.PrivateKey) == 0 || + Config.Sign.PubKey.CurveType == "" || + Config.Sign.SigningPayload == nil || + Config.Sign.SigningPayload.SignatureType == "" { + color.Red("invalid sign input") + } + keyPair, err := keys.ImportPrivateKey(Config.Sign.PrivateKey, Config.Sign.PubKey.CurveType) if err != nil { - fmt.Println(fmt.Errorf("unable to import private keys %#v", err)) + color.Red("unable to import private keys %#v", err) return err } err = keyPair.IsValid() if err != nil { - fmt.Println(fmt.Errorf("keypair invalid with err %#v", err)) + color.Red("keypair invalid with err %#v", err) return err } signer, err := keyPair.Signer() if err != nil { - fmt.Println(fmt.Errorf("signer invalid with err %#v", err)) + color.Red("signer invalid with err %#v", err) return err } @@ -62,11 +69,11 @@ func runSignCmd(_ *cobra.Command, _ []string) error { sign, err := signer.Sign(signingPayload, signatureType) if err != nil { - fmt.Println(fmt.Errorf("unable to sign with err %#v", err)) + color.Red("unable to sign with err %#v", err) return err } hexSig := hex.EncodeToString(sign.Bytes) - color.Blue(hexSig) + color.Green("Signature: %s", hexSig) return nil } diff --git a/cmd/key_verify.go b/cmd/key_verify.go new file mode 100644 index 00000000..838228fd --- /dev/null +++ b/cmd/key_verify.go @@ -0,0 +1,69 @@ +// Copyright 2023 Coinbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package cmd + +import ( + "errors" + "github.com/coinbase/rosetta-sdk-go/keys" + "github.com/fatih/color" + "github.com/spf13/cobra" +) + +var ( + keyVerifyCmd = &cobra.Command{ + Use: "key:verify", + Short: "Verify the signature using the public key", + Long: `Verify the signature using the public key + It supports Keypair specified by https://github.com/coinbase/rosetta-specifications`, + RunE: runKeyVerifyCmd, + } +) + +func runKeyVerifyCmd(_ *cobra.Command, _ []string) error { + if Config.Sign == nil { + return errors.New("sign configuration is missing") + } + + if len(Config.Sign.Signature.Bytes) == 0 || + Config.Sign.SigningPayload == nil || + Config.Sign.SigningPayload.SignatureType == "" || + Config.Sign.PubKey == nil { + color.Red("invalid verify input") + } + + keyPair := keys.KeyPair{ + PublicKey: Config.Sign.PubKey, + } + + signer, err := keyPair.Signer() + if err != nil { + color.Red("signer invalid with err %#v", err) + return err + } + + signature := Config.Sign.Signature + signature.SignatureType = Config.Sign.SigningPayload.SignatureType + signature.SigningPayload = Config.Sign.SigningPayload + signature.PublicKey = Config.Sign.PubKey + + err = signer.Verify(signature) + if err != nil { + color.Red("invalid signature with err %#v", err) + return err + } + + color.Green("Signature Verified.") + return nil +} diff --git a/cmd/root.go b/cmd/root.go index 92bb8b2e..40b930eb 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -376,8 +376,11 @@ default values.`, ) rootCmd.AddCommand(checkSpecCmd) - // Sign command - rootCmd.AddCommand(signCmd) + // Key Sign command + rootCmd.AddCommand(keySignCmd) + + // Key Verify command + rootCmd.AddCommand(keyVerifyCmd) } func initConfig() { diff --git a/configuration/types.go b/configuration/types.go index 8ad3a8f7..f8b2da48 100644 --- a/configuration/types.go +++ b/configuration/types.go @@ -482,6 +482,7 @@ type SignConfiguration struct { PubKey *types.PublicKey `json:"pub_key"` PrivateKey string `json:"private_key"` SigningPayload *types.SigningPayload `json:"signing_payload"` + Signature *types.Signature `json:"signature,omitempty"` } // CheckPerfConfiguration configuration for check perf diff --git a/examples/configuration/sign.json b/examples/configuration/sign.json index c79d1f87..def4ebbe 100644 --- a/examples/configuration/sign.json +++ b/examples/configuration/sign.json @@ -7,7 +7,10 @@ "private_key": "", "signing_payload": { "hex_bytes": "370e74254e8cbaa343af3564901456082ec7af967e45ff24ba061233b1a1b04f", - "signature_type": "ecdsa" + "signature_type": "ecdsa", + "account_identifier": { + "address": "dummy" + } } } } \ No newline at end of file diff --git a/examples/configuration/verify.json b/examples/configuration/verify.json new file mode 100644 index 00000000..9554d13c --- /dev/null +++ b/examples/configuration/verify.json @@ -0,0 +1,18 @@ +{ + "sign": { + "pub_key": { + "curve_type": "secp256k1", + "hex_bytes": "03c7e625aa08cad8f257d9ee2b9b7a0214f19f981afd5b498c728ad7ed6c0c3df6" + }, + "signing_payload": { + "hex_bytes": "370e74254e8cbaa343af3564901456082ec7af967e45ff24ba061233b1a1b04f", + "signature_type": "ecdsa", + "account_identifier": { + "address": "dummy" + } + }, + "signature": { + "hex_bytes": "c80547470b7e4d3fc17c988b2244dfebc909b3e9f7fd0c1387763263cc70d16d24f326b9c12ba2ea278164c0b30f128a809585fc503eda43de429aadb9f893ef" + } + } +} \ No newline at end of file From 2d80e984b99f0dfad356925dea8c560be499ca72 Mon Sep 17 00:00:00 2001 From: Madhur Shrimal Date: Mon, 30 Jan 2023 16:06:22 -0800 Subject: [PATCH 374/376] add key gen command (#388) --- Dockerfile | 2 +- cmd/README.md | 6 +++++- cmd/key_gen.go | 56 ++++++++++++++++++++++++++++++++++++++++++++++++++ cmd/root.go | 16 ++++++++++++++- 4 files changed, 77 insertions(+), 3 deletions(-) create mode 100644 cmd/key_gen.go diff --git a/Dockerfile b/Dockerfile index 07d7bbdc..a6bba933 100644 --- a/Dockerfile +++ b/Dockerfile @@ -31,7 +31,7 @@ RUN mkdir -p "$GOPATH/src" "$GOPATH/bin" && chmod -R 777 "$GOPATH" WORKDIR /go/src -ARG VERSION=v0.10.2 +ARG VERSION=v0.10.3 RUN git clone https://github.com/coinbase/rosetta-cli.git && \ cd rosetta-cli && \ git fetch --all --tags && \ diff --git a/cmd/README.md b/cmd/README.md index 01297daf..0c3bd6cf 100644 --- a/cmd/README.md +++ b/cmd/README.md @@ -4,6 +4,11 @@ Rosetta CLI has a key sign tool, which you can use to sign and verify various cu by rosetta-specifications. This should only be used for local development. Never share private keys anywhere. ### Usage +#### Key Generate +``` +rosetta-cli key:gen --curve-type secp256k1 +``` +Curve Type options are specified by [rosetta-specifications](https://github.com/coinbase/rosetta-specifications/blob/master/models/CurveType.yaml) #### Sign ``` rosetta-cli key:sign --configuration-file config.json @@ -28,6 +33,5 @@ Required fields includes - `signing_payload` - `signature` - ### Troubleshoot - `account_identifier` field in `signing_payload` field should've a dummy address for providing valid payload. \ No newline at end of file diff --git a/cmd/key_gen.go b/cmd/key_gen.go new file mode 100644 index 00000000..49834644 --- /dev/null +++ b/cmd/key_gen.go @@ -0,0 +1,56 @@ +// Copyright 2023 Coinbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package cmd + +import ( + "encoding/hex" + "errors" + + "github.com/coinbase/rosetta-sdk-go/keys" + "github.com/coinbase/rosetta-sdk-go/types" + "github.com/fatih/color" + "github.com/spf13/cobra" +) + +var ( + keyGenCmd = &cobra.Command{ + Use: "key:gen", + Short: "Used to generate a public private key pair", + Long: `Used to generate a public private key pair + It supports Keypair specified by https://github.com/coinbase/rosetta-specifications + Please provide valid CurveType`, + RunE: runKeyGenCmd, + } +) + +func runKeyGenCmd(_ *cobra.Command, _ []string) error { + if len(curveType) == 0 { + color.Red("please provide a non-empty curve type") + return errors.New("invalid curve-type string") + } + + curve := types.CurveType(curveType) + + color.Yellow("Generating new %s keypair...", curve) + keyPair, err := keys.GenerateKeypair(curve) + if err != nil { + color.Red("failed to generate keypair with error %#v", err) + } + + color.Green("CurveType: %s", curve) + color.Green("Public Key (hex): %s", hex.EncodeToString(keyPair.PublicKey.Bytes)) + color.Green("Private Key (hex): %s", hex.EncodeToString(keyPair.PrivateKey)) + return nil +} diff --git a/cmd/root.go b/cmd/root.go index 40b930eb..5a1b52e3 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -17,6 +17,7 @@ package cmd import ( "context" "fmt" + "github.com/coinbase/rosetta-sdk-go/types" "log" "os" "os/signal" @@ -101,6 +102,10 @@ var ( // which has caused production incidents in the past. This can be used for both check:data // and check:construction. asserterConfigurationFile string + + // curveType is used to specify curve type to generate a keypair using rosetta-cli key:gen + // command + curveType string ) // rootPreRun is executed before the root command runs and sets up cpu @@ -381,6 +386,15 @@ default values.`, // Key Verify command rootCmd.AddCommand(keyVerifyCmd) + + keyGenCmd.Flags().StringVar( + &curveType, + "curve-type", + string(types.Secp256k1), + "curve type used to generate the public/private keypair", + ) + // Key Gen command + rootCmd.AddCommand(keyGenCmd) } func initConfig() { @@ -494,6 +508,6 @@ var versionCmd = &cobra.Command{ Use: "version", Short: "Print rosetta-cli version", Run: func(cmd *cobra.Command, args []string) { - fmt.Println("v0.10.2") + fmt.Println("v0.10.3") }, } From f89aae32eaa2b9f301fff8a8f92c9b82e138138f Mon Sep 17 00:00:00 2001 From: jingweicb <96205264+jingweicb@users.noreply.github.com> Date: Wed, 1 Feb 2023 11:05:43 -0800 Subject: [PATCH 375/376] update dependence (#389) --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index d39a1e3e..8da18db4 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/coinbase/rosetta-cli go 1.16 require ( - github.com/coinbase/rosetta-sdk-go v0.8.3-0.20230113233127-4c691644d82a + github.com/coinbase/rosetta-sdk-go v0.8.3 github.com/coinbase/rosetta-sdk-go/types v1.0.0 github.com/fatih/color v1.13.0 github.com/google/go-cmp v0.5.6 // indirect diff --git a/go.sum b/go.sum index 190c8492..448c992b 100644 --- a/go.sum +++ b/go.sum @@ -98,8 +98,8 @@ github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDk github.com/cloudflare/cloudflare-go v0.14.0/go.mod h1:EnwdgGMaFOruiPZRFSgn+TsQ3hQ7C/YWzIGLeu5c304= github.com/coinbase/kryptology v1.8.0 h1:Aoq4gdTsJhSU3lNWsD5BWmFSz2pE0GlmrljaOxepdYY= github.com/coinbase/kryptology v1.8.0/go.mod h1:RYXOAPdzOGUe3qlSFkMGn58i3xUA8hmxYHksuq+8ciI= -github.com/coinbase/rosetta-sdk-go v0.8.3-0.20230113233127-4c691644d82a h1:Bym3+QUv9zIij91/4yX3hs5E2TsDOJ6+8egOeF8KBYI= -github.com/coinbase/rosetta-sdk-go v0.8.3-0.20230113233127-4c691644d82a/go.mod h1:ChOHc+BNq7zqJDDkui0DA124GOvlAiRbdgAc1U9GMDQ= +github.com/coinbase/rosetta-sdk-go v0.8.3 h1:IYqd+Ser5NVh0s7p8p2Ir82iCvi75E1l0NH2H4NEr0Y= +github.com/coinbase/rosetta-sdk-go v0.8.3/go.mod h1:ChOHc+BNq7zqJDDkui0DA124GOvlAiRbdgAc1U9GMDQ= github.com/coinbase/rosetta-sdk-go/types v1.0.0 h1:jpVIwLcPoOeCR6o1tU+Xv7r5bMONNbHU7MuEHboiFuA= github.com/coinbase/rosetta-sdk-go/types v1.0.0/go.mod h1:eq7W2TMRH22GTW0N0beDnN931DW0/WOI1R2sdHNHG4c= github.com/consensys/bavard v0.1.8-0.20210406032232-f3452dc9b572/go.mod h1:Bpd0/3mZuaj6Sj+PqrmIquiOKy397AKGThQPaGzNXAQ= From ae92cd613e589539dc66dafaf90108ec35ce11a9 Mon Sep 17 00:00:00 2001 From: jingweicb <96205264+jingweicb@users.noreply.github.com> Date: Fri, 8 Dec 2023 12:32:30 -0800 Subject: [PATCH 376/376] enable target acc checking (#409) --- .github/workflows/ci.yml | 15 +++++++-------- cmd/root.go | 12 ++++++++++++ configuration/types.go | 3 +++ pkg/tester/data.go | 40 +++++++++++++++++++++++++++++++++++++++- 4 files changed, 61 insertions(+), 9 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 330961a2..8fb10535 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -70,12 +70,11 @@ jobs: go-version: ${{ env.go_version }} - run: make test-cover COVERALLS_TOKEN="$COVERALLS_TOKEN" - Salus: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - with: - version: latest - - run: make salus + # Salus: + # runs-on: ubuntu-latest + # steps: + # - uses: actions/checkout@v3 + # with: + # version: latest + # - run: make salus - \ No newline at end of file diff --git a/cmd/root.go b/cmd/root.go index 5a1b52e3..db54134b 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -61,6 +61,7 @@ var ( requestUUID string statusPort uint InfoMetaData string + targetAccount string // Config is the populated *configuration.Configuration from // the configurationFile. If none is provided, this is set @@ -246,6 +247,13 @@ default values.`, "Override online node url in configuration file", ) + checkDataCmd.Flags().StringVar( + &targetAccount, + "target-account", + "", + "Override target account in configuration file", + ) + checkDataCmd.Flags().Int64Var( &startIndex, "start-block", @@ -425,6 +433,10 @@ func initConfig() { Config.Construction.OfflineURL = offlineURL } + if len(targetAccount) != 0 { + Config.TargetAccount = targetAccount + } + // Override start and end syncing index in configuration file when it's explicitly set via CLI if startIndex != -1 { Config.Data.StartIndex = &startIndex diff --git a/configuration/types.go b/configuration/types.go index f8b2da48..4e2ecfeb 100644 --- a/configuration/types.go +++ b/configuration/types.go @@ -362,6 +362,9 @@ type Configuration struct { // OnlineURL is the URL of a Rosetta API implementation in "online mode". OnlineURL string `json:"online_url"` + // TargetAccount will be the only interest account + TargetAccount string `json:"target_account,omitempty"` + // DataDirectory is a folder used to store logs and any data used to perform validation. // The path can be absolute, or it can be relative to where rosetta-cli // binary is being executed. diff --git a/pkg/tester/data.go b/pkg/tester/data.go index 68afd896..1de767a5 100644 --- a/pkg/tester/data.go +++ b/pkg/tester/data.go @@ -149,6 +149,30 @@ func loadAccounts(filePath string) ([]*types.AccountCurrency, error) { return accounts, nil } +// loadAccount is a utility function to parse the []*types.AccountCurrency +// from a string. +func loadAccount(accountAddress string) []*types.AccountCurrency { + if len(accountAddress) == 0 { + return []*types.AccountCurrency{} + } + + accounts := []*types.AccountCurrency{} + accountIndentifier := &types.AccountIdentifier{ + Address: accountAddress, + // You can set other fields of AccountIdentifier here if needed. + } + + // Create an AccountCurrency instance with the Account field set to the created AccountIdentifier. + targetAccount := &types.AccountCurrency{ + Account: accountIndentifier, + // You can set other fields of AccountCurrency here if needed. + } + + accounts = append(accounts, targetAccount) + + return accounts +} + // CloseDatabase closes the database used by DataTester. func (t *DataTester) CloseDatabase(ctx context.Context) { if err := t.database.Close(ctx); err != nil { @@ -249,6 +273,14 @@ func InitializeData( color.Red(err.Error()) return nil, err } + if len(config.TargetAccount) != 0 { + interestingAccounts = loadAccount(config.TargetAccount) + } + + interestingOnly := false + if len(interestingAccounts) != 0 { + interestingOnly = true + } counterStorage := modules.NewCounterStorage(localStore) blockStorage := modules.NewBlockStorage(localStore, config.SerialBlockWorkers) @@ -357,11 +389,17 @@ func InitializeData( counterStorage, historicalBalanceEnabled, exemptAccounts, - false, + interestingOnly, networkOptions.Allow.BalanceExemptions, config.Data.InitialBalanceFetchDisabled, ) + if interestingOnly { + for _, interesinterestingAccount := range interestingAccounts { + balanceStorageHelper.AddInterestingAddress(interesinterestingAccount.Account.Address) + } + } + balanceStorageHandler := processor.NewBalanceStorageHandler( logger, r,