diff --git a/hail/hail/src/is/hail/types/physical/PArray.scala b/hail/hail/src/is/hail/types/physical/PArray.scala index 8ec966eb8c1..a5ed89bbdce 100644 --- a/hail/hail/src/is/hail/types/physical/PArray.scala +++ b/hail/hail/src/is/hail/types/physical/PArray.scala @@ -1,8 +1,5 @@ package is.hail.types.physical -import is.hail.annotations.Annotation -import is.hail.backend.HailStateManager -import is.hail.check.Gen import is.hail.types.virtual.TArray trait PArrayIterator { @@ -17,7 +14,4 @@ abstract class PArray extends PArrayBackedContainer { final protected[physical] val elementRequired = elementType.required def elementIterator(aoff: Long, length: Int): PArrayIterator - - override def genNonmissingValue(sm: HailStateManager): Gen[IndexedSeq[Annotation]] = - Gen.buildableOf[Array](elementType.genValue(sm)).map(x => x: IndexedSeq[Annotation]) } diff --git a/hail/hail/src/is/hail/types/physical/PBaseStruct.scala b/hail/hail/src/is/hail/types/physical/PBaseStruct.scala index 2a6c1e7ca23..fddf0b248f4 100644 --- a/hail/hail/src/is/hail/types/physical/PBaseStruct.scala +++ b/hail/hail/src/is/hail/types/physical/PBaseStruct.scala @@ -1,9 +1,8 @@ package is.hail.types.physical import is.hail.annotations._ -import is.hail.asm4s.{Code, _} +import is.hail.asm4s._ import is.hail.backend.HailStateManager -import is.hail.check.Gen import is.hail.expr.ir.EmitCodeBuilder import is.hail.types.physical.stypes.interfaces.SBaseStructValue import is.hail.utils._ @@ -144,10 +143,4 @@ abstract class PBaseStruct extends PType { override def loadCheapSCode(cb: EmitCodeBuilder, addr: Code[Long]): SBaseStructValue override lazy val containsPointers: Boolean = types.exists(_.containsPointers) - - override def genNonmissingValue(sm: HailStateManager): Gen[Annotation] = - if (types.isEmpty) { - Gen.const(Annotation.empty) - } else - Gen.uniformSequence(types.map(t => t.genValue(sm))).map(a => Annotation(a: _*)) } diff --git a/hail/hail/src/is/hail/types/physical/PDict.scala b/hail/hail/src/is/hail/types/physical/PDict.scala index 246e924bf68..c6cc0996ffd 100644 --- a/hail/hail/src/is/hail/types/physical/PDict.scala +++ b/hail/hail/src/is/hail/types/physical/PDict.scala @@ -1,8 +1,5 @@ package is.hail.types.physical -import is.hail.annotations._ -import is.hail.backend.HailStateManager -import is.hail.check.Gen import is.hail.types.physical.stypes.interfaces.SContainer import is.hail.types.virtual.TDict @@ -15,7 +12,4 @@ abstract class PDict extends PContainer { def sType: SContainer def elementType: PStruct - - override def genNonmissingValue(sm: HailStateManager): Gen[Annotation] = - Gen.buildableOf2[Map](Gen.zip(keyType.genValue(sm), valueType.genValue(sm))) } diff --git a/hail/hail/src/is/hail/types/physical/PSet.scala b/hail/hail/src/is/hail/types/physical/PSet.scala index 810bae84829..e6109bc58f8 100644 --- a/hail/hail/src/is/hail/types/physical/PSet.scala +++ b/hail/hail/src/is/hail/types/physical/PSet.scala @@ -1,13 +1,7 @@ package is.hail.types.physical -import is.hail.annotations._ -import is.hail.backend.HailStateManager -import is.hail.check.Gen import is.hail.types.virtual.TSet abstract class PSet extends PContainer { lazy val virtualType: TSet = TSet(elementType.virtualType) - - override def genNonmissingValue(sm: HailStateManager): Gen[Annotation] = - Gen.buildableOf[Set](elementType.genValue(sm)) } diff --git a/hail/hail/src/is/hail/types/physical/PType.scala b/hail/hail/src/is/hail/types/physical/PType.scala index 533766fad94..73bb1ba1fd2 100644 --- a/hail/hail/src/is/hail/types/physical/PType.scala +++ b/hail/hail/src/is/hail/types/physical/PType.scala @@ -3,14 +3,12 @@ package is.hail.types.physical import is.hail.annotations._ import is.hail.asm4s._ import is.hail.backend.{ExecuteContext, HailStateManager} -import is.hail.check.{Arbitrary, Gen} import is.hail.expr.ir._ import is.hail.types.{tcoerce, Requiredness} import is.hail.types.physical.stypes.{SType, SValue} import is.hail.types.physical.stypes.concrete.SRNGState import is.hail.types.virtual._ import is.hail.utils._ -import is.hail.variant.ReferenceGenome import org.apache.spark.sql.Row import org.json4s.CustomSerializer @@ -31,108 +29,6 @@ class PStructSerializer extends CustomSerializer[PStruct](format => ) object PType { - def genScalar(required: Boolean): Gen[PType] = - Gen.oneOf( - PBoolean(required), - PInt32(required), - PInt64(required), - PFloat32(required), - PFloat64(required), - PCanonicalString(required), - PCanonicalCall(required), - ) - - val genOptionalScalar: Gen[PType] = genScalar(false) - - val genRequiredScalar: Gen[PType] = genScalar(true) - - def genComplexType(required: Boolean): Gen[PType] = { - val rgDependents = ReferenceGenome.hailReferences.toArray.map(PCanonicalLocus(_, required)) - val others = Array(PCanonicalCall(required)) - Gen.oneOfSeq(rgDependents ++ others) - } - - def genFields(required: Boolean, genFieldType: Gen[PType]): Gen[Array[PField]] = { - Gen.buildableOf[Array]( - Gen.zip(Gen.identifier, genFieldType) - ) - .filter(fields => fields.map(_._1).areDistinct()) - .map(fields => - fields - .iterator - .zipWithIndex - .map { case ((k, t), i) => PField(k, t, i) } - .toArray - ) - } - - def preGenStruct(required: Boolean, genFieldType: Gen[PType]): Gen[PStruct] = - for (fields <- genFields(required, genFieldType)) yield PCanonicalStruct(fields, required) - - def preGenTuple(required: Boolean, genFieldType: Gen[PType]): Gen[PTuple] = - for (fields <- genFields(required, genFieldType)) - yield PCanonicalTuple(required, fields.map(_.typ): _*) - - private val defaultRequiredGenRatio = 0.2 - - def genStruct: Gen[PStruct] = Gen.coin(defaultRequiredGenRatio).flatMap(preGenStruct(_, genArb)) - - val genOptionalStruct: Gen[PType] = preGenStruct(required = false, genArb) - - val genRequiredStruct: Gen[PType] = preGenStruct(required = true, genArb) - - val genInsertableStruct: Gen[PStruct] = Gen.coin(defaultRequiredGenRatio).flatMap(required => - if (required) - preGenStruct(required = true, genArb) - else - preGenStruct(required = false, genOptional) - ) - - def genSized(size: Int, required: Boolean, genPStruct: Gen[PStruct]): Gen[PType] = - if (size < 1) - Gen.const(PCanonicalStruct.empty(required)) - else if (size < 2) - genScalar(required) - else { - Gen.frequency( - (4, genScalar(required)), - (1, genComplexType(required)), - ( - 1, - genArb.map { - PCanonicalArray(_) - }, - ), - ( - 1, - genArb.map { - PCanonicalSet(_) - }, - ), - ( - 1, - genArb.map { - PCanonicalInterval(_) - }, - ), - (1, preGenTuple(required, genArb)), - (1, Gen.zip(genRequired, genArb).map { case (k, v) => PCanonicalDict(k, v) }), - (1, genPStruct.resize(size)), - ) - } - - def preGenArb(required: Boolean, genStruct: Gen[PStruct] = genStruct): Gen[PType] = - Gen.sized(genSized(_, required, genStruct)) - - def genArb: Gen[PType] = Gen.coin(0.2).flatMap(preGenArb(_)) - - val genOptional: Gen[PType] = preGenArb(required = false) - - val genRequired: Gen[PType] = preGenArb(required = true) - - val genInsertable: Gen[PStruct] = genInsertableStruct - - implicit def arbType = Arbitrary(genArb) def canonical(t: Type, required: Boolean, innerRequired: Boolean): PType = { t match { @@ -397,15 +293,6 @@ object PType { } abstract class PType extends Serializable with Requiredness { - self => - - def genValue(sm: HailStateManager): Gen[Annotation] = - if (required) genNonmissingValue(sm) - else Gen.nextCoin(0.05).flatMap(isEmpty => - if (isEmpty) Gen.const(null) else genNonmissingValue(sm) - ) - - def genNonmissingValue(sm: HailStateManager): Gen[Annotation] = virtualType.genNonmissingValue(sm) def virtualType: Type diff --git a/hail/hail/src/is/hail/types/virtual/TArray.scala b/hail/hail/src/is/hail/types/virtual/TArray.scala index c36d7706f61..81a5f4e0113 100644 --- a/hail/hail/src/is/hail/types/virtual/TArray.scala +++ b/hail/hail/src/is/hail/types/virtual/TArray.scala @@ -2,7 +2,6 @@ package is.hail.types.virtual import is.hail.annotations.{Annotation, ExtendedOrdering} import is.hail.backend.HailStateManager -import is.hail.check.Gen import org.json4s.jackson.JsonMethods @@ -43,9 +42,6 @@ final case class TArray(elementType: Type) extends TContainer { override def str(a: Annotation): String = JsonMethods.compact(export(a)) - override def genNonmissingValue(sm: HailStateManager): Gen[IndexedSeq[Annotation]] = - Gen.buildableOf[Array](elementType.genValue(sm)).map(x => x: IndexedSeq[Annotation]) - def mkOrdering(sm: HailStateManager, missingEqual: Boolean): ExtendedOrdering = ExtendedOrdering.iterableOrdering(elementType.ordering(sm), missingEqual) diff --git a/hail/hail/src/is/hail/types/virtual/TBaseStruct.scala b/hail/hail/src/is/hail/types/virtual/TBaseStruct.scala index 1f6f9e6377a..bb79f152988 100644 --- a/hail/hail/src/is/hail/types/virtual/TBaseStruct.scala +++ b/hail/hail/src/is/hail/types/virtual/TBaseStruct.scala @@ -2,7 +2,6 @@ package is.hail.types.virtual import is.hail.annotations._ import is.hail.backend.HailStateManager -import is.hail.check.Gen import is.hail.utils._ import org.apache.spark.sql.Row @@ -95,18 +94,6 @@ abstract class TBaseStruct extends Type { override def str(a: Annotation): String = JsonMethods.compact(export(a)) - override def genNonmissingValue(sm: HailStateManager): Gen[Annotation] = { - if (types.isEmpty) { - Gen.const(Annotation.empty) - } else - Gen.size.flatMap(fuel => - if (types.length > fuel) - Gen.uniformSequence(types.map(t => Gen.const(null))).map(a => Annotation(a: _*)) - else - Gen.uniformSequence(types.map(t => t.genValue(sm))).map(a => Annotation(a: _*)) - ) - } - override def valuesSimilar(a1: Annotation, a2: Annotation, tolerance: Double, absolute: Boolean) : Boolean = a1 == a2 || (a1 != null && a2 != null diff --git a/hail/hail/src/is/hail/types/virtual/TBinary.scala b/hail/hail/src/is/hail/types/virtual/TBinary.scala index 3751ee2244d..d904f538c41 100644 --- a/hail/hail/src/is/hail/types/virtual/TBinary.scala +++ b/hail/hail/src/is/hail/types/virtual/TBinary.scala @@ -2,17 +2,12 @@ package is.hail.types.virtual import is.hail.annotations._ import is.hail.backend.HailStateManager -import is.hail.check.Arbitrary._ -import is.hail.check.Gen case object TBinary extends Type { def _toPretty = "Binary" def _typeCheck(a: Any): Boolean = a.isInstanceOf[Array[Byte]] - override def genNonmissingValue(sm: HailStateManager): Gen[Annotation] = - Gen.buildableOf(arbitrary[Byte]) - def mkOrdering(sm: HailStateManager, _missingEqual: Boolean = true): ExtendedOrdering = ExtendedOrdering.iterableOrdering(new ExtendedOrdering { val missingEqual = _missingEqual diff --git a/hail/hail/src/is/hail/types/virtual/TBoolean.scala b/hail/hail/src/is/hail/types/virtual/TBoolean.scala index 7eaba727d91..ac5f75cb2cf 100644 --- a/hail/hail/src/is/hail/types/virtual/TBoolean.scala +++ b/hail/hail/src/is/hail/types/virtual/TBoolean.scala @@ -2,8 +2,6 @@ package is.hail.types.virtual import is.hail.annotations._ import is.hail.backend.HailStateManager -import is.hail.check.Arbitrary._ -import is.hail.check.Gen case object TBoolean extends Type { def _toPretty = "Boolean" @@ -17,8 +15,6 @@ case object TBoolean extends Type { def parse(s: String): Annotation = s.toBoolean - override def genNonmissingValue(sm: HailStateManager): Gen[Annotation] = arbitrary[Boolean] - override def mkOrdering(sm: HailStateManager, missingEqual: Boolean): ExtendedOrdering = ExtendedOrdering.extendToNull(implicitly[Ordering[Boolean]], missingEqual) diff --git a/hail/hail/src/is/hail/types/virtual/TCall.scala b/hail/hail/src/is/hail/types/virtual/TCall.scala index 1fcb637bd45..d59bbc0d17e 100644 --- a/hail/hail/src/is/hail/types/virtual/TCall.scala +++ b/hail/hail/src/is/hail/types/virtual/TCall.scala @@ -2,7 +2,6 @@ package is.hail.types.virtual import is.hail.annotations._ import is.hail.backend.HailStateManager -import is.hail.check.Gen import is.hail.variant.Call case object TCall extends Type { @@ -15,8 +14,6 @@ case object TCall extends Type { def _typeCheck(a: Any): Boolean = a.isInstanceOf[Int] - override def genNonmissingValue(sm: HailStateManager): Gen[Annotation] = Call.genNonmissingValue - override def str(a: Annotation): String = if (a == null) "NA" else Call.toString(a.asInstanceOf[Call]) diff --git a/hail/hail/src/is/hail/types/virtual/TDict.scala b/hail/hail/src/is/hail/types/virtual/TDict.scala index 54e124db563..0a4f4d9c9a7 100644 --- a/hail/hail/src/is/hail/types/virtual/TDict.scala +++ b/hail/hail/src/is/hail/types/virtual/TDict.scala @@ -2,7 +2,6 @@ package is.hail.types.virtual import is.hail.annotations.{Annotation, ExtendedOrdering} import is.hail.backend.HailStateManager -import is.hail.check.Gen import is.hail.utils._ import org.json4s.jackson.JsonMethods @@ -59,9 +58,6 @@ final case class TDict(keyType: Type, valueType: Type) extends TContainer { override def str(a: Annotation): String = JsonMethods.compact(export(a)) - override def genNonmissingValue(sm: HailStateManager): Gen[Annotation] = - Gen.buildableOf2[Map](Gen.zip(keyType.genValue(sm), valueType.genValue(sm))) - override def valuesSimilar(a1: Annotation, a2: Annotation, tolerance: Double, absolute: Boolean) : Boolean = a1 == a2 || (a1 != null && a2 != null && diff --git a/hail/hail/src/is/hail/types/virtual/TFloat32.scala b/hail/hail/src/is/hail/types/virtual/TFloat32.scala index b2aaf5776de..81d6784512d 100644 --- a/hail/hail/src/is/hail/types/virtual/TFloat32.scala +++ b/hail/hail/src/is/hail/types/virtual/TFloat32.scala @@ -2,8 +2,6 @@ package is.hail.types.virtual import is.hail.annotations._ import is.hail.backend.HailStateManager -import is.hail.check.Arbitrary._ -import is.hail.check.Gen import is.hail.utils._ case object TFloat32 extends TNumeric { @@ -19,9 +17,6 @@ case object TFloat32 extends TNumeric { override def str(a: Annotation): String = if (a == null) "NA" else "%.5e".format(a.asInstanceOf[Float]) - override def genNonmissingValue(sm: HailStateManager): Gen[Annotation] = - arbitrary[Double].map(_.toFloat) - override def valuesSimilar(a1: Annotation, a2: Annotation, tolerance: Double, absolute: Boolean) : Boolean = a1 == a2 || (a1 != null && a2 != null && { diff --git a/hail/hail/src/is/hail/types/virtual/TFloat64.scala b/hail/hail/src/is/hail/types/virtual/TFloat64.scala index 7a8f78a7826..20bc1843611 100644 --- a/hail/hail/src/is/hail/types/virtual/TFloat64.scala +++ b/hail/hail/src/is/hail/types/virtual/TFloat64.scala @@ -2,8 +2,6 @@ package is.hail.types.virtual import is.hail.annotations._ import is.hail.backend.HailStateManager -import is.hail.check.Arbitrary._ -import is.hail.check.Gen import is.hail.utils._ case object TFloat64 extends TNumeric { @@ -19,8 +17,6 @@ case object TFloat64 extends TNumeric { override def str(a: Annotation): String = if (a == null) "NA" else "%.5e".format(a.asInstanceOf[Double]) - override def genNonmissingValue(sm: HailStateManager): Gen[Annotation] = arbitrary[Double] - override def valuesSimilar(a1: Annotation, a2: Annotation, tolerance: Double, absolute: Boolean) : Boolean = a1 == a2 || (a1 != null && a2 != null && { diff --git a/hail/hail/src/is/hail/types/virtual/TInt32.scala b/hail/hail/src/is/hail/types/virtual/TInt32.scala index 9c33fef567a..64ce28795a2 100644 --- a/hail/hail/src/is/hail/types/virtual/TInt32.scala +++ b/hail/hail/src/is/hail/types/virtual/TInt32.scala @@ -2,8 +2,6 @@ package is.hail.types.virtual import is.hail.annotations._ import is.hail.backend.HailStateManager -import is.hail.check.Arbitrary._ -import is.hail.check.Gen case object TInt32 extends TIntegral { def _toPretty = "Int32" @@ -13,8 +11,6 @@ case object TInt32 extends TIntegral { def _typeCheck(a: Any): Boolean = a.isInstanceOf[Int] - override def genNonmissingValue(sm: HailStateManager): Gen[Annotation] = arbitrary[Int] - override def mkOrdering(sm: HailStateManager, missingEqual: Boolean): ExtendedOrdering = ExtendedOrdering.extendToNull(implicitly[Ordering[Int]], missingEqual) diff --git a/hail/hail/src/is/hail/types/virtual/TInt64.scala b/hail/hail/src/is/hail/types/virtual/TInt64.scala index d1557cea240..4979bf53117 100644 --- a/hail/hail/src/is/hail/types/virtual/TInt64.scala +++ b/hail/hail/src/is/hail/types/virtual/TInt64.scala @@ -2,8 +2,6 @@ package is.hail.types.virtual import is.hail.annotations._ import is.hail.backend.HailStateManager -import is.hail.check.Arbitrary._ -import is.hail.check.Gen case object TInt64 extends TIntegral { def _toPretty = "Int64" @@ -13,8 +11,6 @@ case object TInt64 extends TIntegral { def _typeCheck(a: Any): Boolean = a.isInstanceOf[Long] - override def genNonmissingValue(sm: HailStateManager): Gen[Annotation] = arbitrary[Long] - override def mkOrdering(sm: HailStateManager, missingEqual: Boolean): ExtendedOrdering = ExtendedOrdering.extendToNull(implicitly[Ordering[Long]], missingEqual) diff --git a/hail/hail/src/is/hail/types/virtual/TInterval.scala b/hail/hail/src/is/hail/types/virtual/TInterval.scala index 2761b0944a6..50c1a1394ab 100644 --- a/hail/hail/src/is/hail/types/virtual/TInterval.scala +++ b/hail/hail/src/is/hail/types/virtual/TInterval.scala @@ -1,8 +1,7 @@ package is.hail.types.virtual -import is.hail.annotations.{Annotation, ExtendedOrdering} +import is.hail.annotations.ExtendedOrdering import is.hail.backend.HailStateManager -import is.hail.check.Gen import is.hail.utils.{FastSeq, Interval} case class TInterval(pointType: Type) extends Type { @@ -28,9 +27,6 @@ case class TInterval(pointType: Type) extends Type { pointType.typeCheck(i.start) && pointType.typeCheck(i.end) } - override def genNonmissingValue(sm: HailStateManager): Gen[Annotation] = - Interval.gen(pointType.ordering(sm), pointType.genValue(sm)) - override def mkOrdering(sm: HailStateManager, missingEqual: Boolean): ExtendedOrdering = Interval.ordering(pointType.ordering(sm), startPrimary = true, missingEqual) diff --git a/hail/hail/src/is/hail/types/virtual/TLocus.scala b/hail/hail/src/is/hail/types/virtual/TLocus.scala index 4dd1c9986d6..fba034f4a10 100644 --- a/hail/hail/src/is/hail/types/virtual/TLocus.scala +++ b/hail/hail/src/is/hail/types/virtual/TLocus.scala @@ -2,7 +2,6 @@ package is.hail.types.virtual import is.hail.annotations._ import is.hail.backend.HailStateManager -import is.hail.check._ import is.hail.utils._ import is.hail.variant._ @@ -34,9 +33,6 @@ case class TLocus(rgName: String) extends Type { def _typeCheck(a: Any): Boolean = a.isInstanceOf[Locus] - override def genNonmissingValue(sm: HailStateManager): Gen[Annotation] = - Locus.gen(sm.referenceGenomes(rgName)) - override def mkOrdering(sm: HailStateManager, missingEqual: Boolean = true): ExtendedOrdering = sm.referenceGenomes(rgName).extendedLocusOrdering diff --git a/hail/hail/src/is/hail/types/virtual/TNDArray.scala b/hail/hail/src/is/hail/types/virtual/TNDArray.scala index 09873847a05..94e9d35c55e 100644 --- a/hail/hail/src/is/hail/types/virtual/TNDArray.scala +++ b/hail/hail/src/is/hail/types/virtual/TNDArray.scala @@ -2,7 +2,6 @@ package is.hail.types.virtual import is.hail.annotations.{Annotation, ExtendedOrdering, NDArray} import is.hail.backend.HailStateManager -import is.hail.check.Gen import is.hail.expr.{Nat, NatBase} object TNDArray { @@ -111,8 +110,6 @@ final case class TNDArray(elementType: Type, nDimsBase: NatBase) extends Type { case _ => false } - override def genNonmissingValue(sm: HailStateManager): Gen[Annotation] = ??? - override def mkOrdering(sm: HailStateManager, missingEqual: Boolean): ExtendedOrdering = null lazy val shapeType: TTuple = TTuple(Array.fill(nDims)(TInt64): _*) diff --git a/hail/hail/src/is/hail/types/virtual/TRNGState.scala b/hail/hail/src/is/hail/types/virtual/TRNGState.scala index 268e04d9178..7e33f8427a9 100644 --- a/hail/hail/src/is/hail/types/virtual/TRNGState.scala +++ b/hail/hail/src/is/hail/types/virtual/TRNGState.scala @@ -1,8 +1,6 @@ package is.hail.types.virtual -import is.hail.annotations.Annotation import is.hail.backend.HailStateManager -import is.hail.check.Gen case object TRNGState extends Type { override def _toPretty = "RNGState" @@ -10,7 +8,6 @@ case object TRNGState extends Type { override def pyString(sb: StringBuilder): Unit = sb.append("rng_state") - override def genNonmissingValue(sm: HailStateManager): Gen[Annotation] = ??? def _typeCheck(a: Any): Boolean = ??? def mkOrdering(sm: HailStateManager, missingEqual: Boolean) diff --git a/hail/hail/src/is/hail/types/virtual/TSet.scala b/hail/hail/src/is/hail/types/virtual/TSet.scala index 3faf59213fb..9d126d827d1 100644 --- a/hail/hail/src/is/hail/types/virtual/TSet.scala +++ b/hail/hail/src/is/hail/types/virtual/TSet.scala @@ -2,7 +2,6 @@ package is.hail.types.virtual import is.hail.annotations.{Annotation, ExtendedOrdering} import is.hail.backend.HailStateManager -import is.hail.check.Gen import org.json4s.jackson.JsonMethods @@ -46,9 +45,6 @@ final case class TSet(elementType: Type) extends TContainer { override def str(a: Annotation): String = JsonMethods.compact(export(a)) - override def genNonmissingValue(sm: HailStateManager): Gen[Annotation] = - Gen.buildableOf[Set](elementType.genValue(sm)) - override def valueSubsetter(subtype: Type): Any => Any = { assert(elementType == subtype.asInstanceOf[TSet].elementType) identity diff --git a/hail/hail/src/is/hail/types/virtual/TStream.scala b/hail/hail/src/is/hail/types/virtual/TStream.scala index f9ff78a6724..ee9678ca2d1 100644 --- a/hail/hail/src/is/hail/types/virtual/TStream.scala +++ b/hail/hail/src/is/hail/types/virtual/TStream.scala @@ -2,7 +2,6 @@ package is.hail.types.virtual import is.hail.annotations.{Annotation, ExtendedOrdering} import is.hail.backend.HailStateManager -import is.hail.check.Gen import org.json4s.jackson.JsonMethods @@ -38,9 +37,6 @@ final case class TStream(elementType: Type) extends TIterable { override def isRealizable = false - override def genNonmissingValue(sm: HailStateManager): Gen[Annotation] = - throw new UnsupportedOperationException("Streams don't have associated annotations.") - override def mkOrdering(sm: HailStateManager, missingEqual: Boolean): ExtendedOrdering = throw new UnsupportedOperationException("Stream comparison is currently undefined.") diff --git a/hail/hail/src/is/hail/types/virtual/TString.scala b/hail/hail/src/is/hail/types/virtual/TString.scala index 2805f809b79..637cbafd340 100644 --- a/hail/hail/src/is/hail/types/virtual/TString.scala +++ b/hail/hail/src/is/hail/types/virtual/TString.scala @@ -2,8 +2,6 @@ package is.hail.types.virtual import is.hail.annotations._ import is.hail.backend.HailStateManager -import is.hail.check.Arbitrary._ -import is.hail.check.Gen case object TString extends Type { def _toPretty = "String" @@ -15,8 +13,6 @@ case object TString extends Type { def _typeCheck(a: Any): Boolean = a.isInstanceOf[String] - override def genNonmissingValue(sm: HailStateManager): Gen[Annotation] = arbitrary[String] - override def mkOrdering(sm: HailStateManager, missingEqual: Boolean): ExtendedOrdering = ExtendedOrdering.extendToNull(implicitly[Ordering[String]], missingEqual) diff --git a/hail/hail/src/is/hail/types/virtual/TUnion.scala b/hail/hail/src/is/hail/types/virtual/TUnion.scala index 35e65dd8838..ba3c24c9ec2 100644 --- a/hail/hail/src/is/hail/types/virtual/TUnion.scala +++ b/hail/hail/src/is/hail/types/virtual/TUnion.scala @@ -1,8 +1,7 @@ package is.hail.types.virtual -import is.hail.annotations.{Annotation, ExtendedOrdering} +import is.hail.annotations.ExtendedOrdering import is.hail.backend.HailStateManager -import is.hail.check.Gen import is.hail.expr.ir.IRParser import is.hail.utils._ @@ -123,8 +122,6 @@ final case class TUnion(cases: IndexedSeq[Case]) extends Type { } } - override def genNonmissingValue(sm: HailStateManager): Gen[Annotation] = ??? - override def mkOrdering(sm: HailStateManager, missingEqual: Boolean): ExtendedOrdering = ??? override def isIsomorphicTo(t: Type): Boolean = diff --git a/hail/hail/src/is/hail/types/virtual/TVariable.scala b/hail/hail/src/is/hail/types/virtual/TVariable.scala index 5cd3be972df..a395ab02c26 100644 --- a/hail/hail/src/is/hail/types/virtual/TVariable.scala +++ b/hail/hail/src/is/hail/types/virtual/TVariable.scala @@ -1,8 +1,7 @@ package is.hail.types.virtual -import is.hail.annotations.{Annotation, ExtendedOrdering} +import is.hail.annotations.ExtendedOrdering import is.hail.backend.HailStateManager -import is.hail.check.Gen import is.hail.types.Box import scala.collection.mutable @@ -70,8 +69,6 @@ final case class TVariable(name: String, cond: String = null) extends Type { t } - override def genNonmissingValue(sm: HailStateManager): Gen[Annotation] = ??? - override def mkOrdering(sm: HailStateManager, missingEqual: Boolean): ExtendedOrdering = null override def isIsomorphicTo(t: Type): Boolean = diff --git a/hail/hail/src/is/hail/types/virtual/TVoid.scala b/hail/hail/src/is/hail/types/virtual/TVoid.scala index 0c11cb56d9b..ff04792d562 100644 --- a/hail/hail/src/is/hail/types/virtual/TVoid.scala +++ b/hail/hail/src/is/hail/types/virtual/TVoid.scala @@ -1,8 +1,7 @@ package is.hail.types.virtual -import is.hail.annotations.{Annotation, ExtendedOrdering} +import is.hail.annotations.ExtendedOrdering import is.hail.backend.HailStateManager -import is.hail.check.Gen case object TVoid extends Type { override def _toPretty = "Void" @@ -10,8 +9,6 @@ case object TVoid extends Type { override def pyString(sb: StringBuilder): Unit = sb.append("void") - def genNonmissingValue(sm: HailStateManager): Gen[Annotation] = ??? - override def mkOrdering(sm: HailStateManager, missingEqual: Boolean): ExtendedOrdering = null override def _typeCheck(a: Any): Boolean = a.isInstanceOf[Unit] diff --git a/hail/hail/src/is/hail/types/virtual/Type.scala b/hail/hail/src/is/hail/types/virtual/Type.scala index 89212ddeb42..a2497422150 100644 --- a/hail/hail/src/is/hail/types/virtual/Type.scala +++ b/hail/hail/src/is/hail/types/virtual/Type.scala @@ -2,12 +2,10 @@ package is.hail.types.virtual import is.hail.annotations._ import is.hail.backend.HailStateManager -import is.hail.check.{Arbitrary, Gen} import is.hail.expr.{JSONAnnotationImpex, SparkAnnotationImpex} import is.hail.expr.ir._ import is.hail.utils import is.hail.utils._ -import is.hail.variant.ReferenceGenome import org.apache.spark.sql.types.DataType import org.json4s.{CustomSerializer, JValue} @@ -20,96 +18,6 @@ class TypeSerializer extends CustomSerializer[Type](_ => ) ) -object Type { - def genScalar(): Gen[Type] = - Gen.oneOf(TBoolean, TInt32, TInt64, TFloat32, - TFloat64, TString, TCall) - - def genComplexType(): Gen[Type] = { - val rgDependents = ReferenceGenome.hailReferences.toArray.map(TLocus(_)) - val others = Array(TCall) - Gen.oneOfSeq(rgDependents ++ others) - } - - def genFields(genFieldType: Gen[Type]): Gen[Array[Field]] = { - Gen.buildableOf[Array]( - Gen.zip(Gen.identifier, genFieldType) - ) - .filter(fields => fields.map(_._1).areDistinct()) - .map(fields => - fields - .iterator - .zipWithIndex - .map { case ((k, t), i) => Field(k, t, i) } - .toArray - ) - } - - def preGenStruct(genFieldType: Gen[Type]): Gen[TStruct] = - for (fields <- genFields(genFieldType)) yield TStruct(fields) - - def preGenTuple(genFieldType: Gen[Type]): Gen[TTuple] = - for (fields <- genFields(genFieldType)) yield TTuple(fields.map(_.typ): _*) - - private val defaultRequiredGenRatio = 0.2 - def genStruct: Gen[TStruct] = Gen.coin(defaultRequiredGenRatio).flatMap(c => preGenStruct(genArb)) - - def genSized(size: Int, genTStruct: Gen[TStruct]): Gen[Type] = - if (size < 1) - Gen.const(TStruct.empty) - else if (size < 2) - genScalar() - else { - Gen.frequency( - (4, genScalar()), - (1, genComplexType()), - ( - 1, - genArb.map { - TArray(_) - }, - ), - ( - 1, - genArb.map { - TSet(_) - }, - ), - ( - 1, - genArb.map { - TInterval(_) - }, - ), - (1, preGenTuple(genArb)), - (1, Gen.zip(genRequired, genArb).map { case (k, v) => TDict(k, v) }), - (1, genTStruct.resize(size)), - ) - } - - def preGenArb(genStruct: Gen[TStruct] = genStruct): Gen[Type] = - Gen.sized(genSized(_, genStruct)) - - def genArb: Gen[Type] = preGenArb() - - val genOptional: Gen[Type] = preGenArb() - - val genRequired: Gen[Type] = preGenArb() - - def genWithValue(sm: HailStateManager): Gen[(Type, Annotation)] = for { - s <- Gen.size - // prefer smaller type and bigger values - fraction <- Gen.choose(0.1, 0.3) - x = (fraction * s).toInt - y = s - x - t <- Type.genStruct.resize(x) - v <- t.genValue(sm).resize(y) - } yield (t, v) - - implicit def arbType: Arbitrary[Type] = - Arbitrary(genArb) -} - abstract class Type extends VType with Serializable { def children: IndexedSeq[Type] = FastSeq() @@ -177,11 +85,6 @@ abstract class Type extends VType with Serializable { override def toJSON: JValue = JString(toString) - def genNonmissingValue(sm: HailStateManager): Gen[Annotation] - - def genValue(sm: HailStateManager): Gen[Annotation] = - Gen.nextCoin(0.05).flatMap(isEmpty => if (isEmpty) Gen.const(null) else genNonmissingValue(sm)) - def isRealizable: Boolean = children.forall(_.isRealizable) /* compare values for equality, but compare Float and Double values by the absolute value of their diff --git a/hail/hail/src/is/hail/utils/Interval.scala b/hail/hail/src/is/hail/utils/Interval.scala index 6ffa27745d4..3beaa10b34d 100644 --- a/hail/hail/src/is/hail/utils/Interval.scala +++ b/hail/hail/src/is/hail/utils/Interval.scala @@ -1,7 +1,6 @@ package is.hail.utils import is.hail.annotations._ -import is.hail.check._ import is.hail.types.virtual.TBoolean import org.apache.spark.sql.Row @@ -208,16 +207,6 @@ object Interval { IntervalEndpoint(end, if (includesEnd) 1 else -1), ) - def gen[P](pord: ExtendedOrdering, pgen: Gen[P]): Gen[Interval] = - Gen.zip(pgen, pgen, Gen.coin(), Gen.coin()) - .filter { case (x, y, s, e) => pord.compare(x, y) != 0 || (s && e) } - .map { case (x, y, s, e) => - if (pord.compare(x, y) < 0) - Interval(x, y, s, e) - else - Interval(y, x, s, e) - } - def ordering(pord: ExtendedOrdering, startPrimary: Boolean, _missingEqual: Boolean = true) : ExtendedOrdering = new ExtendedOrdering { val missingEqual = _missingEqual diff --git a/hail/hail/src/is/hail/utils/package.scala b/hail/hail/src/is/hail/utils/package.scala index 201cf59aa75..8b6affd9801 100644 --- a/hail/hail/src/is/hail/utils/package.scala +++ b/hail/hail/src/is/hail/utils/package.scala @@ -1,7 +1,6 @@ package is.hail import is.hail.annotations.ExtendedOrdering -import is.hail.check.Gen import is.hail.expr.ir.ByteArrayBuilder import is.hail.io.fs.{FS, FileListEntry} @@ -324,8 +323,6 @@ package object utils def flushDouble(a: Double): Double = if (math.abs(a) < java.lang.Double.MIN_NORMAL) 0.0 else a - def genBase: Gen[Char] = Gen.oneOf('A', 'C', 'T', 'G') - def getPartNumber(fname: String): Int = { val partRegex = """.*/?part-(\d+).*""".r @@ -346,11 +343,6 @@ package object utils } } - // ignore size; atomic, like String - def genDNAString: Gen[String] = Gen.stringOf(genBase) - .resize(12) - .filter(s => !s.isEmpty) - def prettyIdentifier(str: String): String = if (str.matches("""[_a-zA-Z]\w*""")) str diff --git a/hail/hail/src/is/hail/variant/Call.scala b/hail/hail/src/is/hail/variant/Call.scala index 400d9e63559..e542a325bab 100644 --- a/hail/hail/src/is/hail/variant/Call.scala +++ b/hail/hail/src/is/hail/variant/Call.scala @@ -1,6 +1,5 @@ package is.hail.variant -import is.hail.check.Gen import is.hail.expr.Parser import is.hail.utils._ @@ -468,53 +467,4 @@ object Call extends Serializable { } } } - - def check(c: Call, nAlleles: Int): Unit = { - (ploidy(c): @switch) match { - case 0 => - case 1 => - val a = alleleByIndex(c, 0) - assert(a >= 0 && a < nAlleles) - case 2 => - val nGenotypes = triangle(nAlleles) - val udtn = - if (isPhased(c)) { - val p = allelePair(c) - unphasedDiploidGtIndex(Call2(AllelePair.j(p), AllelePair.k(p))) - } else - unphasedDiploidGtIndex(c) - assert( - udtn < nGenotypes, - s"Invalid call found '${c.toString}' for number of alleles equal to '$nAlleles'.", - ) - case _ => - alleles(c).foreach(a => assert(a >= 0 && a < nAlleles)) - } - } - - def gen( - nAlleles: Int, - ploidyGen: Gen[Int] = Gen.choose(0, 2), - phasedGen: Gen[Boolean] = Gen.nextCoin(0.5), - ): Gen[Call] = for { - ploidy <- ploidyGen - phased <- phasedGen - alleles <- Gen.buildableOfN[Array](ploidy, Gen.choose(0, nAlleles - 1)) - } yield { - val c = CallN(alleles, phased) - check(c, nAlleles) - c - } - - def genUnphasedDiploid(nAlleles: Int): Gen[Call] = gen(nAlleles, Gen.const(2), Gen.const(false)) - - def genPhasedDiploid(nAlleles: Int): Gen[Call] = gen(nAlleles, Gen.const(2), Gen.const(true)) - - def genNonmissingValue: Gen[Call] = for { - nAlleles <- Gen.choose(2, 5) - c <- gen(nAlleles) - } yield { - check(c, nAlleles) - c - } } diff --git a/hail/hail/src/is/hail/variant/Genotype.scala b/hail/hail/src/is/hail/variant/Genotype.scala index 3acf5989c6c..d59ae0241ad 100644 --- a/hail/hail/src/is/hail/variant/Genotype.scala +++ b/hail/hail/src/is/hail/variant/Genotype.scala @@ -1,9 +1,7 @@ package is.hail.variant import is.hail.annotations.Annotation -import is.hail.check.Gen import is.hail.types.virtual.{TArray, TCall, TInt32, TStruct} -import is.hail.utils._ import org.apache.spark.sql.Row @@ -240,96 +238,4 @@ object Genotype { else diploidGtIndex(i, j) - def genExtremeNonmissing(nAlleles: Int): Gen[Annotation] = { - val m = Int.MaxValue / (nAlleles + 1) - val nGenotypes = triangle(nAlleles) - val gg = for { - c: Option[Call] <- Gen.option(Call.genUnphasedDiploid(nAlleles)) - ad <- Gen.option(Gen.buildableOfN[Array](nAlleles, Gen.choose(0, m))) - dp <- Gen.option(Gen.choose(0, m)) - gq <- Gen.option(Gen.choose(0, 10000)) - pl <- Gen.oneOfGen( - Gen.option(Gen.buildableOfN[Array](nGenotypes, Gen.choose(0, m))), - Gen.option(Gen.buildableOfN[Array](nGenotypes, Gen.choose(0, 100))), - ) - } yield { - c.foreach(c => pl.foreach(pla => pla(Call.unphasedDiploidGtIndex(c)) = 0)) - pl.foreach { pla => - val m = pla.min - var i = 0 - while (i < pla.length) { - pla(i) -= m - i += 1 - } - } - val g = Annotation( - c.orNull, - ad.map(a => a: IndexedSeq[Int]).orNull, - dp.map(_ + ad.map(_.sum).getOrElse(0)).orNull, - gq.orNull, - pl.map(a => a: IndexedSeq[Int]).orNull, - ) - g - } - gg - } - - def genExtreme(nAlleles: Int): Gen[Annotation] = - Gen.frequency( - (100, genExtremeNonmissing(nAlleles)), - (1, Gen.const(null)), - ) - - def genRealisticNonmissing(nAlleles: Int): Gen[Annotation] = { - val nGenotypes = triangle(nAlleles) - val gg = for { - callRate <- Gen.choose(0d, 1d) - alleleFrequencies <- - Gen.buildableOfN[Array](nAlleles, Gen.choose(1e-6, 1d)) // avoid divison by 0 - .map { rawWeights => - val sum = rawWeights.sum - rawWeights.map(_ / sum) - } - c <- Gen.option( - Gen.zip(Gen.chooseWithWeights(alleleFrequencies), Gen.chooseWithWeights(alleleFrequencies)) - .map { case (gti, gtj) => Call2(gti, gtj) }, - callRate, - ) - ad <- Gen.option(Gen.buildableOfN[Array](nAlleles, Gen.choose(0, 50))) - dp <- Gen.choose(0, 30).map(d => ad.map(o => o.sum + d)) - pl <- Gen.option(Gen.buildableOfN[Array](nGenotypes, Gen.choose(0, 1000)).map { arr => - c match { - case Some(x) => - arr(Call.unphasedDiploidGtIndex(x)) = 0 - arr - case None => - val min = arr.min - arr.map(_ - min) - } - }) - gq <- Gen.choose(-30, 30).map(i => pl.map(pls => math.max(0, gqFromPL(pls) + i))) - } yield Annotation(c.orNull, ad.map(a => a: IndexedSeq[Int]).orNull, dp.orNull, gq.orNull, pl.map(a => a: IndexedSeq[Int]).orNull) - gg - } - - def genRealistic(nAlleles: Int): Gen[Annotation] = - Gen.frequency( - (100, genRealisticNonmissing(nAlleles)), - (1, Gen.const(null)), - ) - - def genGenericCallAndProbabilitiesGenotype(nAlleles: Int): Gen[Annotation] = { - val nGenotypes = triangle(nAlleles) - val gg = for (gp <- Gen.option(Gen.partition(nGenotypes, 32768))) yield { - val c = gp.flatMap(a => Option(uniqueMaxIndex(a))).map(Call2.fromUnphasedDiploidGtIndex(_)) - Row( - c.orNull, - gp.map(gpx => gpx.map(p => p.toDouble / 32768): IndexedSeq[Double]).orNull, - ) - } - Gen.frequency( - (100, gg), - (1, Gen.const(null)), - ) - } } diff --git a/hail/hail/src/is/hail/variant/Locus.scala b/hail/hail/src/is/hail/variant/Locus.scala index da7dc169e7b..0dce76a33c3 100644 --- a/hail/hail/src/is/hail/variant/Locus.scala +++ b/hail/hail/src/is/hail/variant/Locus.scala @@ -1,7 +1,6 @@ package is.hail.variant import is.hail.annotations.Annotation -import is.hail.check.Gen import is.hail.expr.Parser import is.hail.utils._ @@ -34,11 +33,6 @@ object Locus { def fromRow(r: Row): Locus = Locus(r.getAs[String](0), r.getInt(1)) - def gen(rg: ReferenceGenome): Gen[Locus] = for { - (contig, length) <- Contig.gen(rg) - pos <- Gen.choose(1, length) - } yield Locus(contig, pos) - def parse(str: String, rg: ReferenceGenome): Locus = { val elts = str.split(":") val size = elts.length diff --git a/hail/hail/src/is/hail/variant/ReferenceGenome.scala b/hail/hail/src/is/hail/variant/ReferenceGenome.scala index 1aa5a14d526..88e07e04073 100644 --- a/hail/hail/src/is/hail/variant/ReferenceGenome.scala +++ b/hail/hail/src/is/hail/variant/ReferenceGenome.scala @@ -2,7 +2,6 @@ package is.hail.variant import is.hail.annotations.ExtendedOrdering import is.hail.backend.ExecuteContext -import is.hail.check.Gen import is.hail.expr.{ JSONExtractContig, JSONExtractIntervalLocus, JSONExtractReferenceGenome, Parser, } @@ -607,33 +606,6 @@ object ReferenceGenome { Integer.compare(l1.position, l2.position) } - def gen: Gen[ReferenceGenome] = - for { - name <- Gen.identifier.filter(!ReferenceGenome.hailReferences.contains(_)) - nContigs <- Gen.choose(3, 10) - contigs <- Gen.distinctBuildableOfN[Array](nContigs, Gen.identifier) - lengths <- Gen.buildableOfN[Array](nContigs, Gen.choose(1000000, 500000000)) - contigsIndex = contigs.zip(lengths).toMap - xContig <- Gen.oneOfSeq(contigs) - parXA <- Gen.choose(0, contigsIndex(xContig)) - parXB <- Gen.choose(0, contigsIndex(xContig)) - yContig <- Gen.oneOfSeq(contigs) if yContig != xContig - parYA <- Gen.choose(0, contigsIndex(yContig)) - parYB <- Gen.choose(0, contigsIndex(yContig)) - mtContig <- Gen.oneOfSeq(contigs) if mtContig != xContig && mtContig != yContig - } yield ReferenceGenome( - name, - contigs, - contigs.zip(lengths).toMap, - Set(xContig), - Set(yContig), - Set(mtContig), - Array( - (Locus(xContig, math.min(parXA, parXB)), Locus(xContig, math.max(parXA, parXB))), - (Locus(yContig, math.min(parYA, parYB)), Locus(yContig, math.max(parYA, parYB))), - ), - ) - def apply( name: String, contigs: Array[String], diff --git a/hail/hail/src/is/hail/variant/VariantMethods.scala b/hail/hail/src/is/hail/variant/VariantMethods.scala index 96f5cc016bb..63cd4d2c792 100644 --- a/hail/hail/src/is/hail/variant/VariantMethods.scala +++ b/hail/hail/src/is/hail/variant/VariantMethods.scala @@ -1,13 +1,7 @@ package is.hail.variant -import is.hail.annotations.Annotation -import is.hail.check.Gen import is.hail.utils._ -object Contig { - def gen(rg: ReferenceGenome): Gen[(String, Int)] = Gen.oneOfSeq(rg.lengths.toSeq) -} - object VariantMethods { def parse(str: String, rg: ReferenceGenome): (Locus, IndexedSeq[String]) = { @@ -70,48 +64,3 @@ object VariantMethods { } } } - -object VariantSubgen { - def random(rg: ReferenceGenome): VariantSubgen = VariantSubgen( - contigGen = Contig.gen(rg), - nAllelesGen = Gen.frequency((5, Gen.const(2)), (1, Gen.choose(2, 10))), - refGen = genDNAString, - altGen = Gen.frequency((10, genDNAString), (1, Gen.const("*"))), - ) - - def plinkCompatible(rg: ReferenceGenome): VariantSubgen = { - val r = random(rg) - val compatible = (1 until 22).map(_.toString).toSet - r.copy( - contigGen = r.contigGen.filter { case (contig, _) => - compatible.contains(contig) - } - ) - } - - def biallelic(rg: ReferenceGenome): VariantSubgen = random(rg).copy(nAllelesGen = Gen.const(2)) - - def plinkCompatibleBiallelic(rg: ReferenceGenome): VariantSubgen = - plinkCompatible(rg).copy(nAllelesGen = Gen.const(2)) -} - -case class VariantSubgen( - contigGen: Gen[(String, Int)], - nAllelesGen: Gen[Int], - refGen: Gen[String], - altGen: Gen[String], -) { - - def genLocusAlleles: Gen[Annotation] = - for { - (contig, length) <- contigGen - start <- Gen.choose(1, length) - nAlleles <- nAllelesGen - ref <- refGen - altAlleles <- Gen.distinctBuildableOfN[Array]( - nAlleles - 1, - altGen, - ) - .filter(!_.contains(ref)) - } yield Annotation(Locus(contig, start), (ref +: altAlleles).toFastSeq) -} diff --git a/hail/hail/test/src/is/hail/annotations/StagedConstructorSuite.scala b/hail/hail/test/src/is/hail/annotations/StagedConstructorSuite.scala index 8acfb0089c4..3c65bc1042e 100644 --- a/hail/hail/test/src/is/hail/annotations/StagedConstructorSuite.scala +++ b/hail/hail/test/src/is/hail/annotations/StagedConstructorSuite.scala @@ -2,6 +2,7 @@ package is.hail.annotations import is.hail.HailSuite import is.hail.asm4s._ +import is.hail.check.Arbitrary.arbitrary import is.hail.check.{Gen, Prop} import is.hail.expr.ir.{EmitCode, EmitFunctionBuilder, IEmitCode, RequirednessSuite} import is.hail.types.physical._ @@ -10,7 +11,6 @@ import is.hail.types.physical.stypes.interfaces._ import is.hail.types.physical.stypes.primitives.SInt32Value import is.hail.types.virtual._ import is.hail.utils._ - import org.apache.spark.sql.Row import org.testng.annotations.Test @@ -482,10 +482,11 @@ class StagedConstructorSuite extends HailSuite { } @Test def testDeepCopy(): Unit = { - val g = Type.genStruct - .flatMap(t => Gen.zip(Gen.const(t), t.genValue(sm))) - .filter { case (_, a) => a != null } - .map { case (t, a) => (PType.canonical(t).asInstanceOf[PStruct], a) } + val g = for { + tstruct <- arbitrary[TStruct] + struct <- genVal(tstruct) + if struct != null + } yield (PType.canonical(tstruct), struct) val p = Prop.forAll(g) { case (t, a) => assert(t.virtualType.typeCheck(a)) diff --git a/hail/hail/test/src/is/hail/annotations/UnsafeSuite.scala b/hail/hail/test/src/is/hail/annotations/UnsafeSuite.scala index 81a2a82e3a7..dd5d5b4e1e6 100644 --- a/hail/hail/test/src/is/hail/annotations/UnsafeSuite.scala +++ b/hail/hail/test/src/is/hail/annotations/UnsafeSuite.scala @@ -2,7 +2,6 @@ package is.hail.annotations import is.hail.HailSuite import is.hail.backend.ExecuteContext -import is.hail.check._ import is.hail.io._ import is.hail.rvd.AbstractRVDSpec import is.hail.types.physical._ diff --git a/hail/hail/src/is/hail/check/Arbitrary.scala b/hail/hail/test/src/is/hail/check/Arbitrary.scala similarity index 100% rename from hail/hail/src/is/hail/check/Arbitrary.scala rename to hail/hail/test/src/is/hail/check/Arbitrary.scala diff --git a/hail/hail/src/is/hail/check/Gen.scala b/hail/hail/test/src/is/hail/check/Gen.scala similarity index 96% rename from hail/hail/src/is/hail/check/Gen.scala rename to hail/hail/test/src/is/hail/check/Gen.scala index ae157bf802a..be6a9f8eb94 100644 --- a/hail/hail/src/is/hail/check/Gen.scala +++ b/hail/hail/test/src/is/hail/check/Gen.scala @@ -537,36 +537,26 @@ object Gen { def sized[T](f: (Int) => Gen[T]): Gen[T] = Gen((p: Parameters) => f(p.size)(p)) - def applyGen[T, S](gf: Gen[(T) => S], gx: Gen[T]): Gen[S] = Gen { p => + def applyGen[T, S](gf: Gen[T => S], gx: Gen[T]): Gen[S] = Gen { p => val f = gf(p) val x = gx(p) f(x) } } -class Gen[+T](val gen: (Parameters) => T) extends AnyVal { - - def apply(p: Parameters): T = gen(p) +class Gen[+T](val apply: Parameters => T) extends AnyVal { def sample(): T = apply(Parameters.default) - def map[U](f: (T) => U): Gen[U] = Gen(p => f(apply(p))) + def map[U](f: T => U): Gen[U] = Gen(p => f(apply(p))) - def flatMap[U](f: (T) => Gen[U]): Gen[U] = Gen(p => f(apply(p))(p)) + def flatMap[U](f: T => Gen[U]): Gen[U] = Gen(p => f(apply(p))(p)) def resize(newSize: Int): Gen[T] = Gen((p: Parameters) => apply(p.copy(size = newSize))) + + def withFilter(f: T => Boolean): Gen[T] = + Gen((p: Parameters) => Stream.continually(apply(p)).takeWhile(f).head) - // FIXME should be non-strict - def withFilter(f: (T) => Boolean): Gen[T] = Gen { (p: Parameters) => - var x = apply(p) - var i = 0 - while (!f(x)) { - assert(i < 100) - x = apply(p) - i += 1 - } - x - } - - def filter(f: (T) => Boolean): Gen[T] = withFilter(f) + def filter(f: T => Boolean): Gen[T] = + withFilter(f) } diff --git a/hail/hail/src/is/hail/check/Prop.scala b/hail/hail/test/src/is/hail/check/Prop.scala similarity index 100% rename from hail/hail/src/is/hail/check/Prop.scala rename to hail/hail/test/src/is/hail/check/Prop.scala diff --git a/hail/hail/test/src/is/hail/io/IndexBTreeSuite.scala b/hail/hail/test/src/is/hail/io/IndexBTreeSuite.scala index 4682309ba0b..2a1690962c9 100644 --- a/hail/hail/test/src/is/hail/io/IndexBTreeSuite.scala +++ b/hail/hail/test/src/is/hail/io/IndexBTreeSuite.scala @@ -1,8 +1,6 @@ package is.hail.io import is.hail.HailSuite -import is.hail.check.Gen._ -import is.hail.check.Prop._ import is.hail.check.Properties import org.testng.annotations.Test diff --git a/hail/hail/test/src/is/hail/linalg/BlockMatrixSuite.scala b/hail/hail/test/src/is/hail/linalg/BlockMatrixSuite.scala index e32d0c7038e..04a3a8387f4 100644 --- a/hail/hail/test/src/is/hail/linalg/BlockMatrixSuite.scala +++ b/hail/hail/test/src/is/hail/linalg/BlockMatrixSuite.scala @@ -10,8 +10,11 @@ import is.hail.expr.ir.defs.{GetField, TableCollect} import is.hail.linalg.BlockMatrix.ops._ import is.hail.types.virtual.{TFloat64, TInt64, TStruct} import is.hail.utils._ - import breeze.linalg.{*, diag, DenseMatrix => BDM, DenseVector => BDV} +import is.hail.check.Arbitrary.arbitrary +import is.hail.check.Gen._ +import is.hail.check.Prop.forAll +import is.hail.check.{Arbitrary, Gen} import org.apache.spark.sql.Row import org.testng.annotations.Test diff --git a/hail/hail/test/src/is/hail/methods/ExprSuite.scala b/hail/hail/test/src/is/hail/methods/ExprSuite.scala index bc16f8b169b..1d8f10befb5 100644 --- a/hail/hail/test/src/is/hail/methods/ExprSuite.scala +++ b/hail/hail/test/src/is/hail/methods/ExprSuite.scala @@ -2,7 +2,6 @@ package is.hail.methods import is.hail.HailSuite import is.hail.backend.HailStateManager -import is.hail.check.Prop._ import is.hail.check.Properties import is.hail.expr._ import is.hail.expr.ir.IRParser diff --git a/hail/hail/test/src/is/hail/methods/LocalLDPruneSuite.scala b/hail/hail/test/src/is/hail/methods/LocalLDPruneSuite.scala index 4be2d10f0bf..63ce2cd71c1 100644 --- a/hail/hail/test/src/is/hail/methods/LocalLDPruneSuite.scala +++ b/hail/hail/test/src/is/hail/methods/LocalLDPruneSuite.scala @@ -3,7 +3,6 @@ package is.hail.methods import is.hail.{HailSuite, TestUtils} import is.hail.annotations.Annotation import is.hail.check.{Gen, Properties} -import is.hail.check.Prop._ import is.hail.expr.ir.{Interpret, MatrixValue, TableValue} import is.hail.utils._ import is.hail.variant._ diff --git a/hail/hail/test/src/is/hail/types/physical/GenInstances.scala b/hail/hail/test/src/is/hail/types/physical/GenInstances.scala new file mode 100644 index 00000000000..63399e65c13 --- /dev/null +++ b/hail/hail/test/src/is/hail/types/physical/GenInstances.scala @@ -0,0 +1,623 @@ +package is.hail.types.physical + +import is.hail.annotations.{Annotation, ExtendedOrdering} +import is.hail.backend.HailStateManager +import is.hail.check.Arbitrary.arbitrary +import is.hail.check.{Arbitrary, Gen} +import is.hail.types.virtual.{ + Field, TArray, TBoolean, TCall, TDict, TFloat32, TFloat64, TInt32, TInt64, TInterval, TLocus, + TSet, TString, TStruct, TTuple, Type, +} +import is.hail.utils.{Interval, genDNAString, triangle, uniqueMaxIndex} +import is.hail.variant.Call.{ + alleleByIndex, allelePair, alleles, isPhased, ploidy, unphasedDiploidGtIndex, +} +import is.hail.variant.Genotype.gqFromPL +import is.hail.variant.{AllelePair, Call, Call2, CallN, Locus, ReferenceGenome} +import org.apache.spark.sql.Row + +import scala.annotation.switch + +trait GenInstances { + + def genScalar(required: Boolean): Gen[PType] = + Gen.oneOf( + PBoolean(required), + PInt32(required), + PInt64(required), + PFloat32(required), + PFloat64(required), + PCanonicalString(required), + PCanonicalCall(required), + ) + + val genOptionalScalar: Gen[PType] = genScalar(false) + + val genRequiredScalar: Gen[PType] = genScalar(true) + + def genComplexType(required: Boolean): Gen[PType] = { + val rgDependents = ReferenceGenome.hailReferences.toArray.map(PCanonicalLocus(_, required)) + val others = Array(PCanonicalCall(required)) + Gen.oneOfSeq(rgDependents ++ others) + } + + def genFields(required: Boolean, genFieldType: Gen[PType]): Gen[Array[PField]] = + Gen.buildableOf[Array]( + Gen.zip(Gen.identifier, genFieldType) + ) + .filter(fields => fields.map(_._1).areDistinct()) + .map(fields => + fields + .iterator + .zipWithIndex + .map { case ((k, t), i) => PField(k, t, i) } + .toArray + ) + + def preGenStruct(required: Boolean, genFieldType: Gen[PType]): Gen[PStruct] = + for (fields <- genFields(required, genFieldType)) yield PCanonicalStruct(fields, required) + + def preGenTuple(required: Boolean, genFieldType: Gen[PType]): Gen[PTuple] = + for (fields <- genFields(required, genFieldType)) + yield PCanonicalTuple(required, fields.map(_.typ): _*) + + private val defaultRequiredGenRatio = 0.2 + + def genStruct: Gen[PStruct] = Gen.coin(defaultRequiredGenRatio).flatMap(preGenStruct(_, genArb)) + + val genOptionalStruct: Gen[PType] = preGenStruct(required = false, genArb) + + val genRequiredStruct: Gen[PType] = preGenStruct(required = true, genArb) + + val genInsertableStruct: Gen[PStruct] = Gen.coin(defaultRequiredGenRatio).flatMap(required => + if (required) + preGenStruct(required = true, genArb) + else + preGenStruct(required = false, genOptional) + ) + + def genSized(size: Int, required: Boolean, genPStruct: Gen[PStruct]): Gen[PType] = + if (size < 1) + Gen.const(PCanonicalStruct.empty(required)) + else if (size < 2) + genScalar(required) + else { + Gen.frequency( + (4, genScalar(required)), + (1, genComplexType(required)), + ( + 1, + genArb.map { + PCanonicalArray(_) + }, + ), + ( + 1, + genArb.map { + PCanonicalSet(_) + }, + ), + ( + 1, + genArb.map { + PCanonicalInterval(_) + }, + ), + (1, preGenTuple(required, genArb)), + (1, Gen.zip(genRequired, genArb).map { case (k, v) => PCanonicalDict(k, v) }), + (1, genPStruct.resize(size)), + ) + } + + def preGenArb(required: Boolean, genStruct: Gen[PStruct] = genStruct): Gen[PType] = + Gen.sized(genSized(_, required, genStruct)) + + def genArb: Gen[PType] = Gen.coin(0.2).flatMap(preGenArb(_)) + + val genOptional: Gen[PType] = preGenArb(required = false) + + val genRequired: Gen[PType] = preGenArb(required = true) + + val genInsertable: Gen[PStruct] = genInsertableStruct + + implicit def arbType: Arbitrary[PType] = Arbitrary(genArb) + + implicit val arbPArray: Arbitrary[PArray] = + Gen { + for { + elem <- arbitrary[PType] + required <- arbitrary[Boolean] + } yield PCanonicalArray(elem, required) + + def genNonmissingValue(sm: HailStateManager): Gen[IndexedSeq[Annotation]] = + Gen.buildableOf[Array](elementType.genValue(sm)).map(x => x: IndexedSeq[Annotation]) + } + + object Contig { + def gen(rg: ReferenceGenome): Gen[(String, Int)] = Gen.oneOfSeq(rg.lengths.toSeq) + } + + object Locus { + def gen(rg: ReferenceGenome): Gen[Locus] = + for { + (contig, length) <- Contig.gen(rg) + pos <- Gen.choose(1, length) + } yield Locus(contig, pos) + } + + object Call { + def check(c: Call, nAlleles: Int): Unit = { + (ploidy(c): @switch) match { + case 0 => + case 1 => + val a = alleleByIndex(c, 0) + assert(a >= 0 && a < nAlleles) + case 2 => + val nGenotypes = triangle(nAlleles) + val udtn = + if (isPhased(c)) { + val p = allelePair(c) + unphasedDiploidGtIndex(Call2(AllelePair.j(p), AllelePair.k(p))) + } else + unphasedDiploidGtIndex(c) + assert( + udtn < nGenotypes, + s"Invalid call found '${c.toString}' for number of alleles equal to '$nAlleles'.", + ) + case _ => + alleles(c).foreach(a => assert(a >= 0 && a < nAlleles)) + } + } + + def gen( + nAlleles: Int, + ploidyGen: Gen[Int] = Gen.choose(0, 2), + phasedGen: Gen[Boolean] = Gen.nextCoin(0.5), + ): Gen[Call] = for { + ploidy <- ploidyGen + phased <- phasedGen + alleles <- Gen.buildableOfN[Array](ploidy, Gen.choose(0, nAlleles - 1)) + } yield { + val c = CallN(alleles, phased) + check(c, nAlleles) + c + } + + def genUnphasedDiploid(nAlleles: Int): Gen[Call] = gen(nAlleles, Gen.const(2), Gen.const(false)) + + def genPhasedDiploid(nAlleles: Int): Gen[Call] = gen(nAlleles, Gen.const(2), Gen.const(true)) + + def genNonmissingValue: Gen[Call] = for { + nAlleles <- Gen.choose(2, 5) + c <- gen(nAlleles) + } yield { + check(c, nAlleles) + c + } + } + + object Genotype { + def genExtremeNonmissing(nAlleles: Int): Gen[Annotation] = { + val m = Int.MaxValue / (nAlleles + 1) + val nGenotypes = triangle(nAlleles) + val gg = for { + c: Option[Call] <- Gen.option(Call.genUnphasedDiploid(nAlleles)) + ad <- Gen.option(Gen.buildableOfN[Array](nAlleles, Gen.choose(0, m))) + dp <- Gen.option(Gen.choose(0, m)) + gq <- Gen.option(Gen.choose(0, 10000)) + pl <- Gen.oneOfGen( + Gen.option(Gen.buildableOfN[Array](nGenotypes, Gen.choose(0, m))), + Gen.option(Gen.buildableOfN[Array](nGenotypes, Gen.choose(0, 100))), + ) + } yield { + c.foreach(c => pl.foreach(pla => pla(Call.unphasedDiploidGtIndex(c)) = 0)) + pl.foreach { pla => + val m = pla.min + var i = 0 + while (i < pla.length) { + pla(i) -= m + i += 1 + } + } + val g = Annotation( + c.orNull, + ad.map(a => a: IndexedSeq[Int]).orNull, + dp.map(_ + ad.map(_.sum).getOrElse(0)).orNull, + gq.orNull, + pl.map(a => a: IndexedSeq[Int]).orNull, + ) + g + } + gg + } + + def genExtreme(nAlleles: Int): Gen[Annotation] = + Gen.frequency( + (100, genExtremeNonmissing(nAlleles)), + (1, Gen.const(null)), + ) + + def genRealisticNonmissing(nAlleles: Int): Gen[Annotation] = { + val nGenotypes = triangle(nAlleles) + val gg = for { + callRate <- Gen.choose(0d, 1d) + alleleFrequencies <- + Gen.buildableOfN[Array](nAlleles, Gen.choose(1e-6, 1d)) // avoid divison by 0 + .map { rawWeights => + val sum = rawWeights.sum + rawWeights.map(_ / sum) + } + c <- Gen.option( + Gen.zip( + Gen.chooseWithWeights(alleleFrequencies), + Gen.chooseWithWeights(alleleFrequencies), + ) + .map { case (gti, gtj) => Call2(gti, gtj) }, + callRate, + ) + ad <- Gen.option(Gen.buildableOfN[Array](nAlleles, Gen.choose(0, 50))) + dp <- Gen.choose(0, 30).map(d => ad.map(o => o.sum + d)) + pl <- Gen.option(Gen.buildableOfN[Array](nGenotypes, Gen.choose(0, 1000)).map { arr => + c match { + case Some(x) => + arr(Call.unphasedDiploidGtIndex(x)) = 0 + arr + case None => + val min = arr.min + arr.map(_ - min) + } + }) + gq <- Gen.choose(-30, 30).map(i => pl.map(pls => math.max(0, gqFromPL(pls) + i))) + } yield Annotation(c.orNull, ad.map(a => a: IndexedSeq[Int]).orNull, dp.orNull, gq.orNull, pl.map(a => a: IndexedSeq[Int]).orNull) + gg + } + + def genRealistic(nAlleles: Int): Gen[Annotation] = + Gen.frequency( + (100, genRealisticNonmissing(nAlleles)), + (1, Gen.const(null)), + ) + + def genGenericCallAndProbabilitiesGenotype(nAlleles: Int): Gen[Annotation] = { + val nGenotypes = triangle(nAlleles) + val gg = for (gp <- Gen.option(Gen.partition(nGenotypes, 32768))) yield { + val c = gp.flatMap(a => Option(uniqueMaxIndex(a))).map(Call2.fromUnphasedDiploidGtIndex(_)) + Row( + c.orNull, + gp.map(gpx => gpx.map(p => p.toDouble / 32768): IndexedSeq[Double]).orNull, + ) + } + Gen.frequency( + (100, gg), + (1, Gen.const(null)), + ) + } + } + + object VariantSubgen { + def random(rg: ReferenceGenome): VariantSubgen = VariantSubgen( + contigGen = Contig.gen(rg), + nAllelesGen = Gen.frequency((5, Gen.const(2)), (1, Gen.choose(2, 10))), + refGen = genDNAString, + altGen = Gen.frequency((10, genDNAString), (1, Gen.const("*"))), + ) + + def plinkCompatible(rg: ReferenceGenome): VariantSubgen = { + val r = random(rg) + val compatible = (1 until 22).map(_.toString).toSet + r.copy( + contigGen = r.contigGen.filter { case (contig, _) => + compatible.contains(contig) + } + ) + } + + def biallelic(rg: ReferenceGenome): VariantSubgen = random(rg).copy(nAllelesGen = Gen.const(2)) + + def plinkCompatibleBiallelic(rg: ReferenceGenome): VariantSubgen = + plinkCompatible(rg).copy(nAllelesGen = Gen.const(2)) + } + + case class VariantSubgen( + contigGen: Gen[(String, Int)], + nAllelesGen: Gen[Int], + refGen: Gen[String], + altGen: Gen[String], + ) { + + def genLocusAlleles: Gen[Annotation] = + for { + (contig, length) <- contigGen + start <- Gen.choose(1, length) + nAlleles <- nAllelesGen + ref <- refGen + altAlleles <- Gen.distinctBuildableOfN[Array]( + nAlleles - 1, + altGen, + ) + .filter(!_.contains(ref)) + } yield Annotation(Locus(contig, start), (ref +: altAlleles).toFastSeq) + } + + object ReferenceGenome { + def gen: Gen[ReferenceGenome] = + for { + name <- Gen.identifier.filter(!ReferenceGenome.hailReferences.contains(_)) + nContigs <- Gen.choose(3, 10) + contigs <- Gen.distinctBuildableOfN[Array](nContigs, Gen.identifier) + lengths <- Gen.buildableOfN[Array](nContigs, Gen.choose(1000000, 500000000)) + contigsIndex = contigs.zip(lengths).toMap + xContig <- Gen.oneOfSeq(contigs) + parXA <- Gen.choose(0, contigsIndex(xContig)) + parXB <- Gen.choose(0, contigsIndex(xContig)) + yContig <- Gen.oneOfSeq(contigs) if yContig != xContig + parYA <- Gen.choose(0, contigsIndex(yContig)) + parYB <- Gen.choose(0, contigsIndex(yContig)) + mtContig <- Gen.oneOfSeq(contigs) if mtContig != xContig && mtContig != yContig + } yield ReferenceGenome( + name, + contigs, + contigs.zip(lengths).toMap, + Set(xContig), + Set(yContig), + Set(mtContig), + Array( + (Locus(xContig, math.min(parXA, parXB)), Locus(xContig, math.max(parXA, parXB))), + (Locus(yContig, math.min(parYA, parYB)), Locus(yContig, math.max(parYA, parYB))), + ), + ) + } + + object Type { + def genScalar(): Gen[Type] = + Gen.oneOf(TBoolean, TInt32, TInt64, TFloat32, + TFloat64, TString, TCall) + + def genComplexType(): Gen[Type] = { + val rgDependents = ReferenceGenome.hailReferences.toArray.map(TLocus(_)) + val others = Array(TCall) + Gen.oneOfSeq(rgDependents ++ others) + } + + def genFields(genFieldType: Gen[Type]): Gen[Array[Field]] = { + Gen.buildableOf[Array]( + Gen.zip(Gen.identifier, genFieldType) + ) + .filter(fields => fields.map(_._1).areDistinct()) + .map(fields => + fields + .iterator + .zipWithIndex + .map { case ((k, t), i) => Field(k, t, i) } + .toArray + ) + } + + def preGenStruct(genFieldType: Gen[Type]): Gen[TStruct] = + for (fields <- genFields(genFieldType)) yield TStruct(fields) + + def preGenTuple(genFieldType: Gen[Type]): Gen[TTuple] = + for (fields <- genFields(genFieldType)) yield TTuple(fields.map(_.typ): _*) + + private val defaultRequiredGenRatio = 0.2 + + def genStruct: Gen[TStruct] = + Gen.coin(defaultRequiredGenRatio).flatMap(c => preGenStruct(genArb)) + + def genSized(size: Int, genTStruct: Gen[TStruct]): Gen[Type] = + if (size < 1) + Gen.const(TStruct.empty) + else if (size < 2) + genScalar() + else { + Gen.frequency( + (4, genScalar()), + (1, genComplexType()), + ( + 1, + genArb.map { + TArray(_) + }, + ), + ( + 1, + genArb.map { + TSet(_) + }, + ), + ( + 1, + genArb.map { + TInterval(_) + }, + ), + (1, preGenTuple(genArb)), + (1, Gen.zip(genRequired, genArb).map { case (k, v) => TDict(k, v) }), + (1, genTStruct.resize(size)), + ) + } + + def preGenArb(genStruct: Gen[TStruct] = genStruct): Gen[Type] = + Gen.sized(genSized(_, genStruct)) + + def genArb: Gen[Type] = preGenArb() + + val genOptional: Gen[Type] = preGenArb() + + val genRequired: Gen[Type] = preGenArb() + + def genWithValue(sm: HailStateManager): Gen[(Type, Annotation)] = for { + s <- Gen.size + // prefer smaller type and bigger values + fraction <- Gen.choose(0.1, 0.3) + x = (fraction * s).toInt + y = s - x + t <- Type.genStruct.resize(x) + v <- t.genValue(sm).resize(y) + } yield (t, v) + + implicit def arbType: Arbitrary[Type] = + Arbitrary(genArb) + } + + object PBaseStruct { + def genNonmissingValue(sm: HailStateManager): Gen[Annotation] = + if (types.isEmpty) { + Gen.const(Annotation.empty) + } else + Gen.uniformSequence(types.map(t => t.genValue(sm))).map(a => Annotation(a: _*)) + } + + object PDict { + def genNonmissingValue(sm: HailStateManager): Gen[Annotation] = + Gen.buildableOf2[Map](Gen.zip(keyType.genValue(sm), valueType.genValue(sm))) + } + + object PSet { + def genNonmissingValue(sm: HailStateManager): Gen[Annotation] = + Gen.buildableOf[Set](elementType.genValue(sm)) + } + + object TInterval { + def genNonmissingValue(sm: HailStateManager): Gen[Annotation] = + Interval.gen(pointType.ordering(sm), pointType.genValue(sm)) + } + + trait Type { + def genNonmissingValue(sm: HailStateManager): Gen[Annotation] + + def genValue(sm: HailStateManager): Gen[Annotation] = + Gen.nextCoin(0.05).flatMap(isEmpty => + if (isEmpty) Gen.const(null) else genNonmissingValue(sm) + ) + } + + trait TRNGState { + def genNonmissingValue(sm: HailStateManager): Gen[Annotation] = ??? + } + + trait TUnion { + def genNonmissingValue(sm: HailStateManager): Gen[Annotation] = ??? + } + + trait TVariable { + def genNonmissingValue(sm: HailStateManager): Gen[Annotation] = ??? + } + + trait TVoid { + def genNonmissingValue(sm: HailStateManager): Gen[Annotation] = ??? + } + + trait TArray { + def genNonmissingValue(sm: HailStateManager): Gen[IndexedSeq[Annotation]] = + Gen.buildableOf[Array](elementType.genValue(sm)).map(x => x: IndexedSeq[Annotation]) + } + + trait TBaseStruct { + def genNonmissingValue(sm: HailStateManager): Gen[Annotation] = + if (types.isEmpty) { + Gen.const(Annotation.empty) + } else + Gen.size.flatMap(fuel => + if (types.length > fuel) + Gen.uniformSequence(types.map(t => Gen.const(null))).map(a => Annotation(a: _*)) + else + Gen.uniformSequence(types.map(t => t.genValue(sm))).map(a => Annotation(a: _*)) + ) + } + + trait TBinary { + def genNonmissingValue(sm: HailStateManager): Gen[Annotation] = + Gen.buildableOf(arbitrary[Byte]) + } + + trait TBoolean { + def genNonmissingValue(sm: HailStateManager): Gen[Annotation] = + arbitrary[Boolean] + } + + trait TCall { + def genNonmissingValue(sm: HailStateManager): Gen[Annotation] = + Call.genNonmissingValue + } + + trait Foo { + def genValue(sm: HailStateManager): Gen[Annotation] = + if (required) genNonmissingValue(sm) + else Gen.nextCoin(0.05).flatMap(isEmpty => + if (isEmpty) Gen.const(null) else genNonmissingValue(sm) + ) + + def genNonmissingValue(sm: HailStateManager): Gen[Annotation] = + virtualType.genNonmissingValue(sm) + } + + trait TDict { + def genNonmissingValue(sm: HailStateManager): Gen[Annotation] = + Gen.buildableOf2[Map](Gen.zip(keyType.genValue(sm), valueType.genValue(sm))) + } + + trait TFloat32 { + def genNonmissingValue(sm: HailStateManager): Gen[Annotation] = + arbitrary[Float] + } + + trait TFloat64 { + def genNonmissingValue(sm: HailStateManager): Gen[Annotation] = + arbitrary[Double] + } + + trait TInt32 { + def genNonmissingValue(sm: HailStateManager): Gen[Annotation] = + arbitrary[Int] + } + + trait TInt64 { + def genNonmissingValue(sm: HailStateManager): Gen[Annotation] = + arbitrary[Long] + } + + trait TLocus { + def genNonmissingValue(sm: HailStateManager): Gen[Annotation] = + Locus.gen(sm.referenceGenomes(rgName)) + } + + trait TNDArray { + def genNonmissingValue(sm: HailStateManager): Gen[Annotation] = ??? + } + + trait TSet { + def genNonmissingValue(sm: HailStateManager): Gen[Annotation] = + Gen.buildableOf[Set](elementType.genValue(sm)) + } + + trait TStream { + def genNonmissingValue(sm: HailStateManager): Gen[Annotation] = + throw new UnsupportedOperationException("Streams don't have associated annotations.") + } + + trait TString { + def genNonmissingValue(sm: HailStateManager): Gen[Annotation] = arbitrary[String] + } + + trait Interval { + def gen[P](pord: ExtendedOrdering, pgen: Gen[P]): Gen[Interval] = + Gen.zip(pgen, pgen, Gen.coin(), Gen.coin()) + .filter { case (x, y, s, e) => pord.compare(x, y) != 0 || (s && e) } + .map { case (x, y, s, e) => + if (pord.compare(x, y) < 0) + Interval(x, y, s, e) + else + Interval(y, x, s, e) + } + } + + object utils { + def genBase: Gen[Char] = Gen.oneOf('A', 'C', 'T', 'G') + + def genDNAString: Gen[String] = Gen.stringOf(genBase) + .resize(12) + .filter(s => !s.isEmpty) + } + +} diff --git a/hail/hail/test/src/is/hail/utils/BinaryHeapSuite.scala b/hail/hail/test/src/is/hail/utils/BinaryHeapSuite.scala index ea1f69313d8..ccb0796ce5d 100644 --- a/hail/hail/test/src/is/hail/utils/BinaryHeapSuite.scala +++ b/hail/hail/test/src/is/hail/utils/BinaryHeapSuite.scala @@ -1,8 +1,6 @@ package is.hail.utils -import is.hail.check.Arbitrary._ import is.hail.check.Gen -import is.hail.check.Prop._ import scala.collection.mutable diff --git a/hail/hail/test/src/is/hail/utils/BitVectorSuite.scala b/hail/hail/test/src/is/hail/utils/BitVectorSuite.scala index af287922264..d6902df2fcb 100644 --- a/hail/hail/test/src/is/hail/utils/BitVectorSuite.scala +++ b/hail/hail/test/src/is/hail/utils/BitVectorSuite.scala @@ -1,8 +1,5 @@ package is.hail.utils -import is.hail.check._ -import is.hail.check.Prop._ - import org.scalatestplus.testng.TestNGSuite import org.testng.annotations.Test diff --git a/hail/hail/test/src/is/hail/variant/GenotypeSuite.scala b/hail/hail/test/src/is/hail/variant/GenotypeSuite.scala index f88eff4b872..a2a057b30fc 100644 --- a/hail/hail/test/src/is/hail/variant/GenotypeSuite.scala +++ b/hail/hail/test/src/is/hail/variant/GenotypeSuite.scala @@ -2,7 +2,6 @@ package is.hail.variant import is.hail.TestUtils import is.hail.check.Gen -import is.hail.check.Prop._ import is.hail.testUtils.Variant import is.hail.utils._ diff --git a/hail/hail/test/src/is/hail/variant/ReferenceGenomeSuite.scala b/hail/hail/test/src/is/hail/variant/ReferenceGenomeSuite.scala index 693a56c6061..05422e43c51 100644 --- a/hail/hail/test/src/is/hail/variant/ReferenceGenomeSuite.scala +++ b/hail/hail/test/src/is/hail/variant/ReferenceGenomeSuite.scala @@ -2,7 +2,6 @@ package is.hail.variant import is.hail.{HailSuite, TestUtils} import is.hail.backend.{ExecuteContext, HailStateManager} -import is.hail.check.Prop._ import is.hail.check.Properties import is.hail.expr.ir.EmitFunctionBuilder import is.hail.io.reference.{FASTAReader, FASTAReaderConfig, LiftOver}