Inm/nyc zn fix 2906 snap to r5

build.gradle

@@ -485,7 +485,7 @@ def logbackConfig = project.findProperty('logbackCfg') ?: "logback_prod.xml"
 // UseParallelGC
 applicationDefaultJvmArgs = ["-Xmx${myAvailableRam}g", "-Xms${halfOfAvailableMem}g",
                              "-XX:+UseParallelGC", "-XX:+UseParallelOldGC", "-XX:MetaspaceSize=150M", "-Djava.awt.headless=true",
-                             "-Dlogback.configurationFile=${logbackConfig}", "-Xss2048k"] + logGC + jmx
+                             "-Dlogback.configurationFile=${logbackConfig}", "-Xss2048k"] + logGC + jmx + jfr
 println(applicationDefaultJvmArgs)
 
 run {

gradle.deploy.properties

@@ -1,10 +1,24 @@
-runName=art/austin-prod-200k-flowCap-0.2-speedScaling-1.1
-beamBranch=AK/#2624-merg-urbansim-with-austin
-beamCommit=71ccedfc043e9d7a6fd4e4b2b843b4d0962214fe
-beamConfigs=test/input/texas/austin-prod-200k.conf
-instanceType=m5d.24xlarge
+runName=inm/NY-200k-baseline-with-2906-snap-to-r5
 
-#c5.9xlarge (36/72)    -> 5 instances -> $1.53 per Hour
+beamBranch=art/new-york-scenario
+beamCommit=c438594ebee52b334e15b0c8a14757fb2da063e0
+#beamConfigs=test/input/newyork/new-york-modes-calibration-1.conf,test/input/newyork/new-york-modes-calibration-2.conf,test/input/newyork/new-york-modes-calibration-3.conf,test/input/newyork/new-york-modes-calibration-4.conf,test/input/newyork/new-york-modes-calibration-5.conf,test/input/newyork/new-york-modes-calibration-6.conf,test/input/newyork/new-york-modes-calibration-7.conf,test/input/newyork/new-york-modes-calibration-8.conf
+
+beamConfigs=test/input/newyork/new-york-PROD-two-iterations-baseline.conf
+
+#instanceType=m5d.24xlarge
+#maxRAM=360g
+
+instanceType=c5d.24xlarge
+maxRAM=180g
+
+#shutdownBehaviour = stop | terminate
+shutdownBehaviour=terminate
+s3Backup=true
+
+#r5.4xlarge (16/128)    -> $1.008 per Hour
+#r5.8xlarge (32/256)    -> $2.016 per Hour
+#c5.9xlarge (36/72)     -> 5 instances -> $1.53 per Hour
 #m4.10xlarge (40/160)   -> 5 -> $2.00 per Hour
 #m5.12xlarge (48/192)   -> 3 instances -> $2.304 per Hour
 #m5d.12xlarge (48/192)  -> 5 -> $2.712 per Hour
@@ -16,6 +30,7 @@ instanceType=m5d.24xlarge
 #r5d.12xlarge (48/384)  -> 3 -> $3.456 per Hour
 #h1.16xlarge (64/256)   -> 1 -> $3.744 per Hour
 
+#c5d.24xlarge (96/192)  ->   -> $4.608 per Hour
 #m5.24xlarge (96/384)   -> 2 -> $4.608 per Hour
 #i3.16xlarge (64/488)   -> 1 -> $4.992 per Hour
 #i3.metal (72/512)      -> 5 -> $4.992 per Hour
@@ -39,10 +54,6 @@ executeClass=beam.sim.RunBeam
 #executeArgs=['--config', 'production/application-sfbay/experimentsFall2018/fall18-calib-run1.conf']
 beamBatch=false
 shutdownWait=15
-# shutdownBehaviour = stop | terminate
-shutdownBehaviour=terminate
-s3Backup=true
-maxRAM=300g
 #storageSize (in GiB) =  any number between 64 and 256
 storageSize=256
 

src/main/java/beam/physsim/cchRoutingAssignment/RoutingFrameworkWrapper.scala

@@ -71,7 +71,7 @@ class DockerRoutingFrameworkWrapper(
 
   private val graphReader: RoutingFrameworkGraphReader = new RoutingFrameworkGraphReaderImpl()
 
-  private val toolDockerImage = "beammodel/routing-framework:1.0"
+  private val toolDockerImage = "beammodel/routing-framework:1.1"
 
   private val basePath = "/routing-framework/Build/Devel"
   private val convertGraphLauncher = s"$basePath/RawData/ConvertGraph"

src/main/java/beam/physsim/jdeqsim/AgentSimToPhysSimPlanConverter.java

@@ -294,7 +294,7 @@ private boolean shouldWritePlans(int iterationNumber) {
     }
 
     private boolean shouldWriteInIteration(int iterationNumber, int interval) {
-        return interval == 1 || (interval > 0 && iterationNumber % interval == 0);
+        return interval == 1 || (interval > 0 && iterationNumber >= interval && iterationNumber % interval == 0);
     }
 
     private void writePhyssimPlans(IterationEndsEvent event) {

src/main/python/calibration/parse_google_result.py

@@ -119,22 +119,23 @@ def parse_travel_time(travel_time_str):
 def parse_travel_distance(travel_distance):
     if isinstance(travel_distance, float) and math.isnan(travel_distance):
         return float('nan')
-    elif "km" in travel_distance:
-        return float(travel_distance.replace('km', '')) * 1000
-    elif "miles" in travel_distance:
-        # Miles to meteres
-        return float(travel_distance.replace('miles', '')) * 1.60934 * 1000
-    elif "mile" in travel_distance:
-        # Miles to meteres
-        return float(travel_distance.replace('mile', '')) * 1.60934 * 1000
-    elif "ft" in travel_distance:
-        # feet to meteres
-        return float(travel_distance.replace('ft', '')) * 0.3048
-    elif "m" in travel_distance:
-        return float(travel_distance.replace('m', ''))
     else:
-        raise Exception("Cannot parse '%s' as travel distance" % ((travel_distance)))
-
+        travel_distance = travel_distance.replace(",", "")
+        if "km" in travel_distance:
+            return float(travel_distance.replace('km', '')) * 1000
+        elif "miles" in travel_distance:
+            # Miles to meteres
+            return float(travel_distance.replace('miles', '')) * 1.60934 * 1000
+        elif "mile" in travel_distance:
+            # Miles to meteres
+            return float(travel_distance.replace('mile', '')) * 1.60934 * 1000
+        elif "ft" in travel_distance:
+            # feet to meteres
+            return float(travel_distance.replace('ft', '')) * 0.3048
+        elif "m" in travel_distance:
+            return float(travel_distance.replace('m', ''))
+        else:
+            raise Exception("Cannot parse '%s' as travel distance" % ((travel_distance)))
 
 def normalize(df):
     df['route_0_travel_distance_meters'] = df['route_0_travel_distance'].apply(lambda x: parse_travel_distance(x))

src/main/resources/beam-template.conf

@@ -271,6 +271,11 @@ beam.agentsim.agents.vehicles.sharedFleets = [
 beam.agentsim.collectEvents = false
 
 beam.agentsim.agents.population.useVehicleSampling = "boolean | false"
+beam.agentsim.agents.population.industryRemovalProbabilty.enabled = "boolean | false"
+beam.agentsim.agents.population.industryRemovalProbabilty.inputFilePath = ""
+# when removeAgent is true then persons will be removed based on input industryProbability csv,
+# when removeAgent is false then remove all work activities from plan
+beam.agentsim.agents.population.industryRemovalProbabilty.removeAgent = "boolean | false"
 
 beam.exchange.scenario {
   # source for now can be "Beam" or "UrbanSim"
@@ -304,6 +309,7 @@ beam.agentsim.scenarios.frequencyAdjustmentFile = ${beam.inputDirectory}"/r5/Fre
 beam.physsim.flowCapacityFactor = "double | 1.0"
 beam.physsim.storageCapacityFactor = "double | 1.0"
 beam.physsim.speedScalingFactor = "double | 1.0"
+beam.physsim.maxLinkLengthToApplySpeedScalingFactor = "double | 50.0"
 beam.physsim.linkStatsWriteInterval = "int | 1"
 beam.physsim.linkStatsBinSize = "int | 3600"
 beam.physsim.ptSampleSize = "double | 1.0"

src/main/scala/beam/agentsim/agents/PersonAgent.scala

@@ -1083,7 +1083,6 @@ class PersonAgent(
         log.warning(s"$id events leading up to this point:\n\t${getLog.mkString("\n\t")}")
       } else if (stateName == PerformingActivity) {
         logger.debug(s"$id is performing Activity at end of simulation")
-        logger.warn(s"Performing Activity at end of simulation")
       } else {
         logger.warn(s"$id has received Finish while in state: ${stateName}")
       }

src/main/scala/beam/agentsim/agents/modalbehaviors/ChoosesMode.scala

@@ -330,8 +330,10 @@ trait ChoosesMode {
             withParking = willRequestDrivingRoute
           )
         case Some(WALK) =>
+          // FIXME: Our previous tour was WALK, but this should not stop us to use RideHail or try transit!
+          // This is important for the secondary activties because you can end-up walking very far (the problem was that whenever the person chooses WALK, he will never use any other mode anymore)
           responsePlaceholders = makeResponsePlaceholders(boundingBox, withRouting = true)
-          makeRequestWith(withTransit = false, Vector(bodyStreetVehicle), withParking = false)
+          makeRequestWith(withTransit = true, Vector(bodyStreetVehicle), withParking = false)
         case Some(WALK_TRANSIT) =>
           responsePlaceholders = makeResponsePlaceholders(boundingBox, withRouting = true)
           makeRequestWith(withTransit = true, Vector(bodyStreetVehicle), withParking = false)
@@ -452,7 +454,7 @@ trait ChoosesMode {
      * Receive and store data needed for choice.
      */
     case Event(
-        theRouterResult @ RoutingResponse(_, requestId, _, _),
+        theRouterResult @ RoutingResponse(_, requestId, _, _, _),
         choosesModeData: ChoosesModeData
         ) if choosesModeData.rideHail2TransitRoutingRequestId.contains(requestId) =>
       theRouterResult.itineraries.view.foreach { resp =>
@@ -964,6 +966,10 @@ trait ChoosesMode {
           .asInstanceOf[AttributesOfIndividual]
       val availableAlts = Some(filteredItinerariesForChoice.map(_.tripClassifier).mkString(":"))
 
+//      if (filteredItinerariesForChoice.size == 1 && filteredItinerariesForChoice.head.tripClassifier == WALK) {
+//        log.info(s"Person ${id}, filteredItinerariesForChoice: ${filteredItinerariesForChoice}")
+//      }
+
       modeChoiceCalculator(
         filteredItinerariesForChoice,
         attributesOfIndividual,

src/main/scala/beam/agentsim/agents/ridehail/RideHailAgent.scala

@@ -322,7 +322,7 @@ class RideHailAgent(
       beamServices.beamRouter ! veh2StallRequest
 //      }
       stay
-    case Event(RoutingResponse(itineraries, _, _, _), data) =>
+    case Event(RoutingResponse(itineraries, _, _, _, _), data) =>
       log.debug("Received routing response, initiating trip to parking stall")
       val theLeg = itineraries.head.beamLegs.head
       val updatedPassengerSchedule = PassengerSchedule().addLegs(Seq(theLeg))
@@ -410,7 +410,7 @@ class RideHailAgent(
     case ev @ Event(ParkingInquiryResponse(_, _), _) =>
       stash()
       stay()
-    case ev @ Event(RoutingResponse(_, _, _, _), _) =>
+    case ev @ Event(RoutingResponse(_, _, _, _, _), _) =>
       stash()
       stay()
   }

src/main/scala/beam/agentsim/agents/ridehail/RideHailManager.scala

@@ -1880,7 +1880,8 @@ class RideHailManager(
             modRHA2Cust,
             rideHailAgent2CustomerResponse.requestId,
             None,
-            isEmbodyWithCurrentTravelTime = false
+            isEmbodyWithCurrentTravelTime = false,
+            rideHailAgent2CustomerResponse.computedInMs
           )
 
         val passengerSchedule = PassengerSchedule().addLegs(

src/main/scala/beam/agentsim/agents/vehicles/BeamVehicle.scala

@@ -412,7 +412,14 @@ object BeamVehicle {
   )
 
   def noSpecialChars(theString: String): String =
-    theString.replaceAll("[\\\\|\\\\^]+", ":")
+    theString
+      .replaceAll("[\\\\|\\\\^]+", ":")
+      .replace("[", "")
+      .replace("]", "")
+      .replace("(", "")
+      .replace(")", "")
+      .replace("/", "")
+      .replace("\\", "")
 
   def createId[A](id: Id[A], prefix: Option[String] = None): Id[BeamVehicle] = {
     createId(id.toString, prefix)

src/main/scala/beam/agentsim/events/handling/TravelTimeGoogleStatistic.scala

@@ -47,6 +47,7 @@ class TravelTimeGoogleStatistic(
   if (cfg.enable && apiKey == null)
     logger.warn("google api key is empty")
   private val queryDate = getQueryDate(cfg.queryDate)
+  logger.info(s"queryDate :$queryDate")
 
   private val enabled = cfg.enable && apiKey != null
   private val constraints: Set[TravelConstraint] = if (cfg.tolls) Set.empty else Set(AvoidTolls)
@@ -64,6 +65,8 @@ class TravelTimeGoogleStatistic(
   override def notifyIterationEnds(event: IterationEndsEvent): Unit = {
     if (enabled
         && cfg.iterationInterval > 0
+        // HACK: to avoid requests at iteration 0
+        && event.getIteration > 0
         && event.getIteration % cfg.iterationInterval == 0) {
       logger.info(
         "Executing google API call for iteration #{}, query date = {}",
@@ -143,7 +146,7 @@ class TravelTimeGoogleStatistic(
               ec.event.endX,
               ec.event.arrivalTime - ec.event.departureTime,
               ec.route.durationIntervalInSeconds,
-              ec.route.durationInTrafficSeconds,
+              ec.route.durationInTrafficSeconds.getOrElse(-1).toString,
               geoUtils.distLatLon2Meters(
                 new Coord(ec.event.startX, ec.event.startY),
                 new Coord(ec.event.endX, ec.event.endY)

src/main/scala/beam/analysis/ModeChoiceAlternativesCollector.scala

@@ -134,7 +134,7 @@ class ModeChoiceAlternativesCollector(beamServices: BeamServices)
 
     csvFilePath = beamServices.matsimServices.getControlerIO.getIterationFilename(
       iteration,
-      "modeChoiceAlternativesWhenRHPooled.csv.gz"
+      "modeChoiceDetailed.csv.gz"
     )
 
     csvWriter = new CsvWriter(

src/main/scala/beam/router/BeamRouter.scala

@@ -481,7 +481,8 @@ object BeamRouter {
     itineraries: Seq[EmbodiedBeamTrip],
     requestId: Int,
     request: Option[RoutingRequest],
-    isEmbodyWithCurrentTravelTime: Boolean
+    isEmbodyWithCurrentTravelTime: Boolean,
+    computedInMs: Long = -1
   )
 
   case class RoutingFailure(cause: Throwable, requestId: Int)