Merge pull request #157 from cybercongress/148_internal_operations

148 internal operations [pump]

build.gradle

@@ -1,24 +1,23 @@
 /* Gradle plugins part */
 buildscript {
     ext {
-        kotlinVersion = "1.2.30"
+        kotlinVersion = "1.2.41"
         kotlinCoroutinesVersion = "0.22.3"
         jacksonVersion = "2.9.2"
-        kafkaVersion = "1.0.0"
+        kafkaVersion = "1.0.1"
         springKafkaVersion = "2.1.4.RELEASE"
         cassandraVersion = "3.5.0"
-        springCassandraVersion = "2.0.6.RELEASE"
-        /* todo migrate to web3j 3.+*/
+        springCassandraVersion = "2.0.7.RELEASE"
         web3jVersion = "3.4.0"
         rxkotlinVersion = "2.2.0"
-        reactorVersion = "3.1.3.RELEASE"
+        reactorVersion = "3.1.7.RELEASE"
         apacheHttpClientVersion = "4.1.3"
         ehcacheVersion = "3.4.0"
         elasticVersion = "5.5.3"
 
         // plugins
         dependencyPluginVersion = "1.0.3.RELEASE"
-        bootPluginVersion = "2.0.0.RELEASE"
+        bootPluginVersion = "2.0.2.RELEASE"
         detektPluginVersion = "1.0.0.RC6-4"
 
         // tests
@@ -126,8 +125,7 @@ subprojects {
             dependency("org.slf4j:slf4j-api:$slf4jVersion")
             dependency("ch.qos.logback:logback-classic:$lockbackVersion")
 
-            dependency("org.jetbrains.kotlin:kotlin-stdlib-jre8:$kotlinVersion")
-            dependency("org.jetbrains.kotlin:kotlin-stdlib-jre7:$kotlinVersion")
+            dependency("org.jetbrains.kotlin:kotlin-stdlib-jdk8:$kotlinVersion")
             dependency("org.jetbrains.kotlin:kotlin-stdlib:$kotlinVersion")
             dependency("org.jetbrains.kotlin:kotlin-reflect:$kotlinVersion")
             dependency("org.jetbrains.kotlinx:kotlinx-coroutines-jdk8:$kotlinCoroutinesVersion")
@@ -167,6 +165,7 @@ subprojects {
             dependency("com.datastax.cassandra:cassandra-driver-extras:$cassandraVersion")
 
             dependency("org.web3j:core:$web3jVersion")
+            dependency("org.web3j:parity:$web3jVersion")
             dependency("io.reactivex.rxjava2:rxkotlin:$rxkotlinVersion") {
                 exclude 'org.jetbrains.kotlin:kotlin-stdlib'
             }
@@ -194,7 +193,7 @@ project(":common") {
         compile("org.slf4j:slf4j-api")
         compile("ch.qos.logback:logback-classic")
 
-        compile("org.jetbrains.kotlin:kotlin-stdlib-jre8")
+        compile("org.jetbrains.kotlin:kotlin-stdlib-jdk8")
         compile("org.jetbrains.kotlin:kotlin-reflect")
 
 

cassandra-service/src/main/kotlin/fund/cyber/cassandra/configuration/CassandraConfiguration.kt

@@ -18,6 +18,7 @@ import org.springframework.data.cassandra.config.AbstractReactiveCassandraConfig
 import org.springframework.data.cassandra.config.CassandraEntityClassScanner
 import org.springframework.data.cassandra.config.CassandraSessionFactoryBean
 import org.springframework.data.cassandra.config.SchemaAction
+import org.springframework.data.cassandra.core.convert.CassandraCustomConversions
 import org.springframework.data.cassandra.core.convert.MappingCassandraConverter
 import org.springframework.data.cassandra.core.cql.keyspace.CreateKeyspaceSpecification
 import org.springframework.data.cassandra.core.mapping.CassandraMappingContext
@@ -32,12 +33,16 @@ val Chain.keyspace: String get() = lowerCaseName
 
 const val REPOSITORY_NAME_DELIMETER = "__"
 
-fun mappingContext(cluster: Cluster, keyspace: String, basePackage: String): CassandraMappingContext {
+fun mappingContext(
+        cluster: Cluster, keyspace: String, basePackage: String,
+        customConversions: CassandraCustomConversions = CassandraCustomConversions(emptyList<Any>())
+): CassandraMappingContext {
 
     val mappingContext = CassandraMappingContext()
 
     mappingContext.setInitialEntitySet(CassandraEntityClassScanner.scan(basePackage))
     mappingContext.setUserTypeResolver(SimpleUserTypeResolver(cluster, keyspace))
+    mappingContext.setCustomConversions(customConversions)
 
     return mappingContext
 }

cassandra-service/src/main/kotlin/fund/cyber/cassandra/ethereum/configuration/Converters.kt

@@ -0,0 +1,21 @@
+package fund.cyber.cassandra.ethereum.configuration
+
+import com.fasterxml.jackson.databind.ObjectMapper
+import fund.cyber.search.model.ethereum.TxTrace
+import org.springframework.core.convert.converter.Converter
+
+/**
+ * Used to create TxTrace from byted json.
+ */
+class TxTraceReadConverter(private val jsonDeserializer: ObjectMapper) : Converter<ByteArray, TxTrace> {
+
+    override fun convert(source: ByteArray) = jsonDeserializer.readValue(source, TxTrace::class.java)!!
+}
+
+/**
+ * Used to convert TxTrace to byted json.
+ */
+class TxTraceWriteConverter(private val jsonSerializer: ObjectMapper) : Converter<TxTrace, ByteArray> {
+
+    override fun convert(source: TxTrace) = jsonSerializer.writeValueAsBytes(source)!!
+}

cassandra-service/src/main/kotlin/fund/cyber/cassandra/ethereum/configuration/EthereumRepositoryConfiguration.kt

@@ -9,15 +9,15 @@ import fund.cyber.cassandra.configuration.REPOSITORY_NAME_DELIMETER
 import fund.cyber.cassandra.configuration.getKeyspaceSession
 import fund.cyber.cassandra.configuration.keyspace
 import fund.cyber.cassandra.configuration.mappingContext
-import fund.cyber.cassandra.ethereum.repository.EthereumContractRepository
 import fund.cyber.cassandra.ethereum.repository.EthereumBlockRepository
+import fund.cyber.cassandra.ethereum.repository.EthereumContractRepository
 import fund.cyber.cassandra.ethereum.repository.EthereumContractTxRepository
 import fund.cyber.cassandra.ethereum.repository.EthereumTxRepository
 import fund.cyber.cassandra.ethereum.repository.EthereumUncleRepository
+import fund.cyber.cassandra.ethereum.repository.PageableEthereumBlockTxRepository
 import fund.cyber.cassandra.ethereum.repository.PageableEthereumContractMinedBlockRepository
 import fund.cyber.cassandra.ethereum.repository.PageableEthereumContractMinedUncleRepository
 import fund.cyber.cassandra.ethereum.repository.PageableEthereumContractTxRepository
-import fund.cyber.cassandra.ethereum.repository.PageableEthereumBlockTxRepository
 import fund.cyber.cassandra.migration.BlockchainMigrationSettings
 import fund.cyber.cassandra.migration.MigrationSettings
 import fund.cyber.search.configuration.CASSANDRA_HOSTS
@@ -26,6 +26,8 @@ import fund.cyber.search.configuration.CASSANDRA_PORT
 import fund.cyber.search.configuration.CASSANDRA_PORT_DEFAULT
 import fund.cyber.search.configuration.CHAIN
 import fund.cyber.search.configuration.env
+import fund.cyber.search.jsonDeserializer
+import fund.cyber.search.jsonSerializer
 import fund.cyber.search.model.chains.EthereumFamilyChain
 import org.springframework.beans.factory.InitializingBean
 import org.springframework.beans.factory.annotation.Autowired
@@ -44,13 +46,15 @@ import org.springframework.data.cassandra.config.ClusterBuilderConfigurer
 import org.springframework.data.cassandra.core.CassandraTemplate
 import org.springframework.data.cassandra.core.ReactiveCassandraOperations
 import org.springframework.data.cassandra.core.ReactiveCassandraTemplate
+import org.springframework.data.cassandra.core.convert.CassandraCustomConversions
 import org.springframework.data.cassandra.core.convert.MappingCassandraConverter
 import org.springframework.data.cassandra.core.cql.keyspace.CreateKeyspaceSpecification
 import org.springframework.data.cassandra.core.cql.session.DefaultBridgedReactiveSession
 import org.springframework.data.cassandra.core.cql.session.DefaultReactiveSessionFactory
 import org.springframework.data.cassandra.repository.config.EnableReactiveCassandraRepositories
 import org.springframework.data.cassandra.repository.support.CassandraRepositoryFactory
 import org.springframework.data.cassandra.repository.support.ReactiveCassandraRepositoryFactory
+import org.springframework.data.convert.CustomConversions
 import org.springframework.stereotype.Component
 
 
@@ -108,6 +112,18 @@ class EthereumRepositoryConfiguration(
         session.setKeyspaceName(keyspaceName)
         return session
     }
+
+    @Bean
+    override fun customConversions(): CustomConversions {
+        return customEthereumConversions()
+    }
+}
+
+private fun customEthereumConversions(): CassandraCustomConversions {
+    val additionConverters = listOf(
+        TxTraceReadConverter(jsonDeserializer), TxTraceWriteConverter(jsonSerializer)
+    )
+    return CassandraCustomConversions(additionConverters)
 }
 
 
@@ -143,7 +159,9 @@ class EthereumRepositoriesConfiguration : InitializingBean {
 
                     //create sessions
                     val converter = MappingCassandraConverter(mappingContext(cluster, keyspace.name,
-                        "fund.cyber.cassandra.ethereum.model"))
+                        "fund.cyber.cassandra.ethereum.model", customEthereumConversions()))
+                    converter.customConversions = customEthereumConversions()
+                    converter.afterPropertiesSet()
                     val session = getKeyspaceSession(cluster, keyspace.name, converter).also { it.afterPropertiesSet() }
                     val reactiveSession = DefaultReactiveSessionFactory(DefaultBridgedReactiveSession(session.`object`))
 

cassandra-service/src/main/kotlin/fund/cyber/cassandra/ethereum/model/Contract.kt

@@ -17,20 +17,21 @@ import java.time.Instant
 
 @Table("contract_summary")
 data class CqlEthereumContractSummary(
-        @PrimaryKey override val hash: String,
-        @Column("confirmed_balance") val confirmedBalance: String,
-        @Column("smart_contract") val smartContract: Boolean,
-        @Column("confirmed_total_received") val confirmedTotalReceived: String,
-        @Column("tx_number") val txNumber: Int,
-        @Column("uncle_number") val minedUncleNumber: Int,
-        @Column("mined_block_number") val minedBlockNumber: Int,
-        @Column("first_activity_date") val firstActivityDate: Instant,
-        @Column("last_activity_date") val lastActivityDate: Instant,
-        override val version: Long,
-        @Column("kafka_delta_offset") override val kafkaDeltaOffset: Long,
-        @Column("kafka_delta_partition") override val kafkaDeltaPartition: Int,
-        @Column("kafka_delta_topic") override val kafkaDeltaTopic: String,
-        @Column("kafka_delta_offset_committed") override val kafkaDeltaOffsetCommitted: Boolean = false
+    @PrimaryKey override val hash: String,
+    @Column("confirmed_balance") val confirmedBalance: String,
+    @Column("smart_contract") val smartContract: Boolean,
+    @Column("confirmed_total_received") val confirmedTotalReceived: String,
+    @Column("tx_number") val txNumber: Int,
+    @Column("successful_operation_number") val successfulOpNumber: Int,
+    @Column("uncle_number") val minedUncleNumber: Int,
+    @Column("mined_block_number") val minedBlockNumber: Int,
+    @Column("first_activity_date") val firstActivityDate: Instant,
+    @Column("last_activity_date") val lastActivityDate: Instant,
+    override val version: Long,
+    @Column("kafka_delta_offset") override val kafkaDeltaOffset: Long,
+    @Column("kafka_delta_partition") override val kafkaDeltaPartition: Int,
+    @Column("kafka_delta_topic") override val kafkaDeltaTopic: String,
+    @Column("kafka_delta_offset_committed") override val kafkaDeltaOffsetCommitted: Boolean = false
 ) : CqlEthereumItem, CqlContractSummary
 
 

cassandra-service/src/main/kotlin/fund/cyber/cassandra/ethereum/model/Operation.kt

@@ -0,0 +1,21 @@
+package fund.cyber.cassandra.ethereum.model
+
+import fund.cyber.search.model.ethereum.OperationType
+import java.math.BigDecimal
+import java.time.Instant
+
+/**
+ * Represent operations, moving eth, for contract.
+ */
+data class CqlEthereumContractOperation(
+    val txHash: String,
+    val txTime: Instant,
+    val contract: String,
+    val type: OperationType,
+    val from: String,
+    val to: String,
+    val value: BigDecimal,
+    val error: String?, //not null if operation execution failed
+    val gasUsed: Long,
+    val gasLimit: Long
+)

cassandra-service/src/main/kotlin/fund/cyber/cassandra/ethereum/model/Transaction.kt

@@ -3,38 +3,44 @@
 package fund.cyber.cassandra.ethereum.model
 
 import fund.cyber.search.model.ethereum.EthereumTx
+import fund.cyber.search.model.ethereum.TxTrace
 import org.springframework.data.cassandra.core.mapping.Column
 import org.springframework.data.cassandra.core.mapping.PrimaryKey
 import org.springframework.data.cassandra.core.mapping.Table
 import java.math.BigDecimal
 import java.time.Instant
 
+
+//todo add tx status
 @Table("tx")
 data class CqlEthereumTx(
-        @PrimaryKey val hash: String,
-        val nonce: Long,
-        @Column("block_hash") val blockHash: String?,
-        @Column("block_number") val blockNumber: Long,
-        @Column("first_seen_time") val firstSeenTime: Instant,
-        @Column("block_time") val blockTime: Instant?,
-        @Column(forceQuote = true) val from: String,
-        @Column(forceQuote = true) val to: String?,
-        val value: String,
-        @Column("gas_price") val gasPrice: BigDecimal,
-        @Column("gas_limit") val gasLimit: Long,
-        @Column("gas_used") val gasUsed: Long,
-        val fee: String,
-        val input: String,
-        @Column("created_contract") val createdContract: String?
+    @PrimaryKey val hash: String,
+    val nonce: Long,
+    @Column("block_hash") val blockHash: String?,
+    @Column("block_number") val blockNumber: Long,
+    @Column("first_seen_time") val firstSeenTime: Instant,
+    @Column("block_time") val blockTime: Instant?,
+    @Column(forceQuote = true) val from: String,
+    @Column(forceQuote = true) val to: String?,
+    val value: String,
+    @Column("gas_price") val gasPrice: BigDecimal,
+    @Column("gas_limit") val gasLimit: Long,
+    @Column("gas_used") val gasUsed: Long,
+    val fee: String,
+    val input: String,
+    @Column("created_contract") val createdContract: String?,
+    @Column("trace_json") val trace: TxTrace?   //saved in cassandra as json string
 ) : CqlEthereumItem {
 
     constructor(tx: EthereumTx) : this(
-            hash = tx.hash, nonce = tx.nonce, blockHash = tx.blockHash, blockNumber = tx.blockNumber,
-            blockTime = tx.blockTime, from = tx.from, to = tx.to, firstSeenTime = tx.firstSeenTime,
-            value = tx.value.toString(), gasPrice = tx.gasPrice, gasLimit = tx.gasLimit, gasUsed = tx.gasUsed,
-            fee = tx.fee.toString(), input = tx.input, createdContract = tx.createdSmartContract
-
+        hash = tx.hash, nonce = tx.nonce, blockHash = tx.blockHash, blockNumber = tx.blockNumber,
+        blockTime = tx.blockTime, from = tx.from, to = tx.to, firstSeenTime = tx.firstSeenTime,
+        value = tx.value.toString(), gasPrice = tx.gasPrice, gasLimit = tx.gasLimit, gasUsed = tx.gasUsed,
+        fee = tx.fee.toString(), input = tx.input, createdContract = tx.createdSmartContract, trace = tx.trace
+        //operations = emptyList()
     )
 
     fun contractsUsedInTransaction() = listOfNotNull(from, to, createdContract)
 }
+
+

cassandra-service/src/main/kotlin/fund/cyber/cassandra/ethereum/repository/UpdateContractSummaryRepository.kt

@@ -28,6 +28,7 @@ interface EthereumUpdateContractSummaryRepository : ReactiveCrudRepository<CqlEt
             smart_contract = :#{#summary.smartContract},
             confirmed_total_received = :#{#summary.confirmedTotalReceived},
             tx_number = :#{#summary.txNumber},
+            successful_operation_number = :#{#summary.successfulOpNumber},
             uncle_number = :#{#summary.minedUncleNumber},
             mined_block_number = :#{#summary.minedBlockNumber},
             last_activity_date = :#{#summary.lastActivityDate},
@@ -48,13 +49,13 @@ interface EthereumUpdateContractSummaryRepository : ReactiveCrudRepository<CqlEt
     @Consistency(value = ConsistencyLevel.LOCAL_QUORUM)
     @Query("""
         INSERT INTO contract_summary (hash, confirmed_balance, smart_contract,
-          confirmed_total_received, tx_number, uncle_number, mined_block_number,
+          confirmed_total_received, tx_number, successful_operation_number, uncle_number, mined_block_number,
           first_activity_date, last_activity_date, version, kafka_delta_offset, kafka_delta_topic,
           kafka_delta_partition, kafka_delta_offset_committed)
         VALUES (:#{#summary.hash}, :#{#summary.confirmedBalance}, :#{#summary.smartContract},
-            :#{#summary.confirmedTotalReceived}, :#{#summary.txNumber}, :#{#summary.minedUncleNumber},
-            :#{#summary.minedBlockNumber}, :#{#summary.firstActivityDate}, :#{#summary.lastActivityDate},
-            :#{#summary.version}, :#{#summary.kafkaDeltaOffset},
+            :#{#summary.confirmedTotalReceived}, :#{#summary.txNumber},:#{#summary.successfulOpNumber},
+            :#{#summary.minedUncleNumber}, :#{#summary.minedBlockNumber}, :#{#summary.firstActivityDate},
+            :#{#summary.lastActivityDate}, :#{#summary.version}, :#{#summary.kafkaDeltaOffset},
             :#{#summary.kafkaDeltaTopic}, :#{#summary.kafkaDeltaPartition},
             false)
         IF NOT EXISTS

cassandra-service/src/main/resources/migrations/ethereum/0_initial.cql

@@ -24,7 +24,6 @@ CREATE TABLE IF NOT exists ethereum.block (
     uncles FROZEN < list < text > >
 );
 
-
 CREATE TABLE IF NOT exists ethereum.tx (
     hash text PRIMARY KEY,
     nonce bigint,
@@ -40,7 +39,8 @@ CREATE TABLE IF NOT exists ethereum.tx (
     gas_used bigint,
     fee text,
     input text,
-    created_contract text
+    created_contract text,
+    trace_json text
 );
 
 CREATE TABLE IF NOT exists ethereum.tx_preview_by_contract (
@@ -75,6 +75,7 @@ CREATE TABLE IF NOT exists ethereum.contract_summary (
     smart_contract boolean,
     confirmed_total_received text,
     tx_number int,
+    successful_operation_number int,
     uncle_number int,
     mined_block_number int,
     first_activity_date timestamp,

cassandra-service/src/main/resources/migrations/ethereum_classic/0_initial.cql

@@ -40,7 +40,8 @@ CREATE TABLE IF NOT exists ethereum_classic.tx (
     gas_used bigint,
     fee text,
     input text,
-    created_contract text
+    created_contract text,
+    trace_json text
 );
 
 CREATE TABLE IF NOT exists ethereum_classic.tx_preview_by_contract (
@@ -74,6 +75,7 @@ CREATE TABLE IF NOT exists ethereum_classic.contract_summary (
     smart_contract boolean,
     confirmed_total_received text,
     tx_number int,
+    successful_operation_number int,
     uncle_number int,
     mined_block_number int,
     first_activity_date timestamp,