Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add logic on newRollUp to pass costMultiplier #769

Merged
merged 3 commits into from
Mar 19, 2021
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
28 changes: 25 additions & 3 deletions core/src/main/scala/com/yahoo/maha/core/fact/Fact.scala
Original file line number Diff line number Diff line change
Expand Up @@ -843,6 +843,23 @@ case class FactBuilder private[fact](private val baseFact: Fact, private var tab
mutableDiscardingSet.toSet
}

private[this] def remapMultiplier(
costMultiplierMap: Map[RequestType, CostMultiplier]
, costMultiplier: Option[BigDecimal] = None): Map[RequestType, CostMultiplier] = {
costMultiplierMap.map(rTypeMult =>
rTypeMult._1 -> {
val adjustedLRL: LongRangeLookup[BigDecimal] =
LongRangeLookup(
rTypeMult._2.rows.list.map(
row => (row._1, row._2 * costMultiplier.getOrElse(1))
)
)
CostMultiplier(adjustedLRL)
}

)
}

def withNewGrain(name: String
, from: String
, grain: Grain
Expand Down Expand Up @@ -1269,7 +1286,8 @@ case class FactBuilder private[fact](private val baseFact: Fact, private var tab
, forceFilters: Set[ForceFilter] = Set.empty
, resetAliasIfNotPresent: Boolean = false
, availableOnwardsDate : Option[String] = None
, underlyingTableName: Option[String] = None) : FactBuilder = {
, underlyingTableName: Option[String] = None
, costMultiplier: Option[BigDecimal] = None) : FactBuilder = {
require(tableMap.nonEmpty, "no table to create subset from")
require(tableMap.contains(from), s"from table not valid $from")
require(!tableMap.contains(name), s"table $name already exists")
Expand Down Expand Up @@ -1329,6 +1347,9 @@ case class FactBuilder private[fact](private val baseFact: Fact, private var tab
case _ =>
ddlAnnotation
}

val remappedMultiplier: Map[RequestType, CostMultiplier] = remapMultiplier(fromTable.costMultiplierMap, costMultiplier)

tableMap = tableMap +
(name -> new FactTable(
name
Expand All @@ -1341,7 +1362,7 @@ case class FactBuilder private[fact](private val baseFact: Fact, private var tab
, Option(fromTable)
, fromTable.annotations
, newDDLAnnotation
, fromTable.costMultiplierMap
, remappedMultiplier
, newForceFilters
, fromTable.defaultCardinality
, fromTable.defaultRowCount
Expand Down Expand Up @@ -1442,6 +1463,7 @@ case class FactBuilder private[fact](private val baseFact: Fact, private var tab

val newGrain = grain.getOrElse(fromTable.grain)
val newForceFilters = if(forceFilters.isEmpty) fromTable.forceFilters else forceFilters
val remappedMultiplier: Map[RequestType, CostMultiplier] = remapMultiplier(fromTable.costMultiplierMap, costMultiplier)

tableMap = tableMap +
(name -> new FactTable(
Expand All @@ -1455,7 +1477,7 @@ case class FactBuilder private[fact](private val baseFact: Fact, private var tab
, Option(fromTable)
, fromTable.annotations ++ overrideAnnotations
, ddlAnnotations
, fromTable.costMultiplierMap
, remappedMultiplier
, newForceFilters
, fromTable.defaultCardinality
, fromTable.defaultRowCount
Expand Down
18 changes: 17 additions & 1 deletion core/src/test/scala/com/yahoo/maha/core/ColumnTest.scala
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
package com.yahoo.maha.core

import com.yahoo.maha.core.dimension.{ConstDimCol, DimCol, HivePartDimCol}
import com.yahoo.maha.core.fact.{ConstFactCol, DruidConstDerFactCol, HiveDerFactCol, NoopRollup}
import com.yahoo.maha.core.fact.{ConstFactCol, DruidConstDerFactCol, HiveDerFactCol, NoopRollup, PostgresDerFactCol}
import org.scalatest.funsuite.AnyFunSuite
import org.scalatest.matchers.should.Matchers

Expand Down Expand Up @@ -69,4 +69,20 @@ class ColumnTest extends AnyFunSuite with Matchers {
}
}
}

test("PostgresDerFactCol test") {
ColumnContext.withColumnContext { implicit cc: ColumnContext =>
import PostgresExpression._
DimCol("dimCol", IntType())
val col = PostgresDerFactCol("postgres_ctr_copywith_test", DecType(), "{clicks}" /- "{impressions}" * "1000")
ColumnContext.withColumnContext {
implicit cc:ColumnContext=>
col.copyWith(cc, Map.empty, true)
}
ColumnContext.withColumnContext {
implicit cc:ColumnContext=>
col.copyWith(cc, Map.empty, false)
}
}
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@ import com.yahoo.maha.core.NoopSchema.NoopSchema
import com.yahoo.maha.core._
import com.yahoo.maha.core.ddl.HiveDDLAnnotation
import com.yahoo.maha.core.dimension.DimCol
import com.yahoo.maha.core.request.{RequestType, SyncRequest}
import com.yahoo.maha.core.request.{AsyncRequest, RequestType, SyncRequest}

/**
* Created by jians on 10/20/15.
Expand Down Expand Up @@ -295,6 +295,18 @@ class NewRollupFactTest extends BaseFactTest {
require(bcOption.isDefined, "Failed to get candidates!")
assert(bcOption.get.facts.values.exists( f => f.fact.name == "fact2") === true)
}

test("Create a new rollup with a different cost basis") {
val fact = fact1
fact.newRollUp("fact2", "fact1", Set("ad_group_id"), schemas = Set(AdvertiserSchema, ResellerSchema), availableOnwardsDate = Some(toDate), costMultiplier = Some(0.5))

val costMultMap1 = publicFact(fact).facts("fact1").costMultiplierMap
val costMultMap2 = publicFact(fact).facts("fact2").costMultiplierMap
assert(costMultMap1(SyncRequest).rows.list.head._2 == 1 && costMultMap2(SyncRequest).rows.list.head._2 == 0.5)
assert(costMultMap1(AsyncRequest).rows.list.head._2 == 1 && costMultMap2(AsyncRequest).rows.list.head._2 == 0.5)
}


}


Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@ import com.yahoo.maha.core.CoreSchema._
import com.yahoo.maha.core.NoopSchema.NoopSchema
import com.yahoo.maha.core.ddl.HiveDDLAnnotation
import com.yahoo.maha.core.dimension.DimCol
import com.yahoo.maha.core.request.SyncRequest
import com.yahoo.maha.core.request.{AsyncRequest, SyncRequest}
import com.yahoo.maha.core.{ColumnContext, DailyGrain, InFilter, InFilterOperation, IntType, OracleEngine, PrimaryKey, StrType}

/**
Expand Down Expand Up @@ -165,4 +165,14 @@ class createSubsetTest extends BaseFactTest {
require(bcOption.isDefined, "Failed to get candidates!")
assert(bcOption.get.facts.keys.exists(_ == "fact2") === true, "create subset failed")
}

test("Create a subset with a different cost basis") {
val fact = fact1
fact.createSubset("fact2", "fact1", Set("clicks"), schemas = Set(AdvertiserSchema, ResellerSchema), availableOnwardsDate = Some(toDate), costMultiplier = Some(0.5))

val costMultMap1 = publicFact(fact).facts("fact1").costMultiplierMap
val costMultMap2 = publicFact(fact).facts("fact2").costMultiplierMap
assert(costMultMap1(SyncRequest).rows.list.head._2 == 1 && costMultMap2(SyncRequest).rows.list.head._2 == 0.5)
assert(costMultMap1(AsyncRequest).rows.list.head._2 == 1 && costMultMap2(AsyncRequest).rows.list.head._2 == 0.5)
}
}