Skip to content
This repository has been archived by the owner on Jan 3, 2023. It is now read-only.

[SQL-DS-CACHE-47]Add plasma native get timeout #56

Merged
merged 1 commit into from
Mar 22, 2021
Merged

[SQL-DS-CACHE-47]Add plasma native get timeout #56

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion ...he/src/main/scala/org/apache/spark/sql/execution/datasources/oap/filecache/OapCache.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1067,7 +1067,7 @@ class ExternalCache(fiberType: FiberType) extends OapCache with Logging {
var fiberCache : FiberCache = null
logDebug(s"Cache hit, get from external cache.")
val plasmaClient = plasmaClientPool(clientRoundRobin.getAndAdd(1) % clientPoolSize)
val buf: ByteBuffer = plasmaClient.getObjAsByteBuffer(objectId, -1, false)
val buf: ByteBuffer = plasmaClient.getObjAsByteBuffer(objectId, 1000, false)
Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

can we make it configurable?

if(buf.asInstanceOf[DirectBuffer].address() == 0) {
logWarning("Get return an invalid value.")
fiberCache = cache(fiberId)
Expand Down
12 changes: 6 additions & 6 deletions Plasma-based-cache/src/test/oap-perf-suite/scala/org/apache/spark/sql/BenchmarkConfig.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,10 +16,9 @@
*/
package org.apache.spark.sql


import scala.collection.mutable
import org.apache.spark.SparkConf

import org.apache.spark.SparkConf

// TODO: use SQLConf style i.e. (value, defaultValue)
class BenchmarkConfig {
Expand Down Expand Up @@ -115,7 +114,7 @@ class BenchmarkConfig {

object BenchmarkConfig {
val INDEX_ENABLE = "oap.benchmark.config.index"
val FILE_FORMAT = "oap.benchmark.config.format"
val FILE_FORMAT = "oap.benchmark.config.format"
}

abstract class BenchmarkConfigSelector {
Expand Down Expand Up @@ -174,8 +173,9 @@ trait ParquetVsOrcConfigSet extends BenchmarkConfigSelector{
// TODO: choose conf
val _conf = new SparkConf()
def allConfigurations: Seq[BenchmarkConfig] =
if(_conf.get("spark.sql.oap.cache.memory.manager") == "offheap" ||
((!_conf.contains("spark.sql.oap.cache.memory.manager")) && _conf.get("spark.sql.oap.fiberCache.memory.manager")=="offheap")) {
if (_conf.get("spark.sql.oap.cache.memory.manager") == "offheap" ||
((!_conf.contains("spark.sql.oap.cache.memory.manager")) &&
_conf.get("spark.sql.oap.fiberCache.memory.manager")=="offheap")) {
Seq(
new BenchmarkConfig()
.setBenchmarkConfName("Orc w/ index")
Expand Down Expand Up @@ -245,7 +245,7 @@ trait ParquetVsOrcConfigSet extends BenchmarkConfigSelector{
.setSparkConf("spark.sql.oap.index.data.cache.separation.enabled", "true")
.setSparkConf("spark.oap.cache.strategy", "mix")
)
}else{
} else {
Seq(
new BenchmarkConfig()
.setBenchmarkConfName("Orc w/ index")
Expand Down