From ba898168deb17f7644086136198455316326256e Mon Sep 17 00:00:00 2001 From: Colin Sheppard Date: Tue, 2 Oct 2018 20:06:00 -0700 Subject: [PATCH] merging 496 back to master (#689) * merging 496 back to master * turn into proper code blocks * Update beam.conf reset beam config * get failing tests back up * touch to get new build * flaky test * touch to rebuild * make await short * ignore single mode spec as on master * analysis scripts * cleaning up sf-light data * change await to 60 sec * updating PhyssimCalcLinkStats * restart maybe flakey test --- build.gradle | 3 +- docs/developers.rst | 14 +- src/main/R/beam-utilities.R | 32 +- src/main/R/bin/csv2Rdata.R | 3 +- src/main/R/bin/exp2plots.R | 121 +++---- src/main/R/bin/exp2ridehail.R | 86 ++++- src/main/R/bin/indiv2exp.R | 4 +- src/main/R/pems-to-network.R | 78 +++++ src/main/R/population.R | 50 +++ src/main/R/viz-plans.R | 32 ++ .../bash/copyAllFilesFromRunningExperiment.sh | 36 +++ src/main/bash/deploy-examples.sh | 29 ++ src/main/bash/listModes.sh | 5 +- src/main/bash/pop-attrib.sh | 16 + .../agentsim/events/PathTraversalEvent.java | 2 +- ...PhyssimCalcLinkSpeedDistributionStats.java | 298 ++++++++++++++++++ .../physsim/PhyssimCalcLinkSpeedStats.java | 22 ++ .../physsim/PhyssimCalcLinkStats.java | 4 +- .../via/EventWriterXML_viaCompatible.java | 9 +- .../AgentSimToPhysSimPlanConverter.java | 34 +- .../java/beam/utils/BeamCalcLinkStats.java | 29 +- src/main/java/beam/utils/UnzipUtility.java | 31 ++ src/main/resources/beam-template.conf | 3 +- .../agents/modalbehaviors/ChoosesMode.scala | 43 +-- .../agents/ridehail/RideHailManager.scala | 2 +- .../beam/calibration/BeamSigoptTuner.scala | 6 +- .../beam/calibration/ExperimentRunner.scala | 22 +- .../example/ModeChoiceObjectiveFunction.scala | 44 ++- .../example/RideHailObjectiveFunction.scala | 24 ++ .../calibration/utils/CreateExperiment.scala | 2 +- .../beam/router/r5/R5RoutingWorker.scala | 75 ++--- src/main/scala/beam/sim/BeamHelper.scala | 3 + src/main/scala/beam/sim/BeamMobsim.scala | 25 +- src/main/scala/beam/sim/BeamWarmStart.scala | 83 +++-- .../scala/beam/sim/config/BeamConfig.scala | 56 ++-- .../plansampling/AvailableModeUtils.scala | 10 +- ...simCalcLinkSpeedDistributionStatsTest.java | 106 +++++++ .../PhyssimCalcLinkSpeedStatsTest.java | 77 +++++ .../physsim/PhyssimCalcLinkStatsTest.java | 8 +- .../physsim/BeamCalcLinkStatsSpec.scala | 2 +- .../scala/beam/integration/ParkingSpec.scala | 2 +- .../router/TimeDependentRoutingSpec.scala | 2 +- .../beam/router/WarmStartRoutingSpec.scala | 12 +- test.csv | 3 + test/input/beamville/beam.conf | 2 + test/input/beamville/lccm-simple.csv | 3 + test/input/equil-square/equil-0.001k.conf | 5 + .../input/sf-light/sample/1k/vehicleTypes.csv | 3 + test/input/sf-light/sf-light-25k.conf | 4 +- test/input/sf-light/sf-light.conf | 35 +- test/input/sf-light/taz-centers.csv | 3 + test/input/sf-light/taz-parking.csv | 3 + 52 files changed, 1323 insertions(+), 283 deletions(-) create mode 100644 src/main/R/pems-to-network.R create mode 100644 src/main/R/population.R create mode 100644 src/main/R/viz-plans.R create mode 100644 src/main/bash/deploy-examples.sh create mode 100644 src/main/bash/pop-attrib.sh create mode 100644 src/main/java/beam/analysis/physsim/PhyssimCalcLinkSpeedDistributionStats.java create mode 100644 src/main/scala/beam/calibration/impl/example/RideHailObjectiveFunction.scala create mode 100755 src/test/java/beam/analysis/physsim/PhyssimCalcLinkSpeedDistributionStatsTest.java create mode 100755 src/test/java/beam/analysis/physsim/PhyssimCalcLinkSpeedStatsTest.java create mode 100644 test.csv create mode 100644 test/input/beamville/lccm-simple.csv create mode 100644 test/input/sf-light/sample/1k/vehicleTypes.csv create mode 100755 test/input/sf-light/taz-centers.csv create mode 100644 test/input/sf-light/taz-parking.csv diff --git a/build.gradle b/build.gradle index d684af38068..6a7b22ba86a 100755 --- a/build.gradle +++ b/build.gradle @@ -259,7 +259,7 @@ scalafmt { // configFilePath = ".scalafmt.conf" // .scalafmt.conf in the project root is default value, provide only if other location is needed } -compileScala.dependsOn(scalafmtAll) +//compileScala.dependsOn(scalafmtAll) // Task to run scala tests, as Scala tests not picked up by Gradle by default. task spec(dependsOn: ['testClasses'], type: JavaExec) { @@ -570,3 +570,4 @@ task execute(type:JavaExec) { } } } + diff --git a/docs/developers.rst b/docs/developers.rst index 32e6423dbb0..c74ec4f8e6a 100755 --- a/docs/developers.rst +++ b/docs/developers.rst @@ -112,17 +112,17 @@ Production versus test data. Any branch beginning with "production" or "applicat However, sometimes troubleshooting / debugging / development happens on a production branch. The cleanest way to get changes to source code or other non-production files back into master is the following. -Checkout your production branch: +Checkout your production branch:: git checkout production-branch -Bring branch even with master +Bring branch even with master:: git merge master Resolve conflicts if needed -Capture the files that are different now between production and master: +Capture the files that are different now between production and master:: git diff --name-only HEAD master > diff-with-master.txt @@ -130,19 +130,19 @@ You have created a file "diff-with-master.txt" containing a listing of every fil IMPORTANT!!!! -- Edit the file diff-with-master.txt and remove all production-related data (this typically will be all files underneath "production" sub-directory. -Checkout master +Checkout master:: git checkout master -Create a new branch off of master, this is where you will stage the files to then merge back into master: +Create a new branch off of master, this is where you will stage the files to then merge back into master:: git checkout -b new-branch-with-changes-4ci -Do a file by file checkout of all differing files from production branch onto master: +Do a file by file checkout of all differing files from production branch onto master:: cat diff-with-master.txt | xargs git checkout production-branch -- -Note, if any of our diffs include the deletion of a file on your production branch, then you will need to remove (i.e. with "git remove" these before you do the above "checkout" step and you should also remove them from the diff-with-master.txt". If you don't do this, you will see an error message ("did not match any file(s) known to git.") and the checkout command will not be completed. +Note, if any of our diffs include the deletion of a file on your production branch, then you will need to remove (i.e. with "git remove" these before you do the above "checkout" step and you should also remove them from the diff-with-master.txt"). If you don't do this, you will see an error message ("did not match any file(s) known to git.") and the checkout command will not be completed. Finally, commit the files that were checked out of the production branch, push, and go create your pull request! diff --git a/src/main/R/beam-utilities.R b/src/main/R/beam-utilities.R index e2f5f3344b1..b71c0bba491 100755 --- a/src/main/R/beam-utilities.R +++ b/src/main/R/beam-utilities.R @@ -3,14 +3,16 @@ load.libraries(c('GEOquery','XML')) clean.and.relabel <- function(ev,factor.to.scale.personal.back,factor.to.scale.transit.back,val.of.time=16.9){ # Clean and relabel - ev[vehicle_type=="bus",vehicle_type:="Bus"] - ev[vehicle_type=="CAR" | substr(vehicle,1,5)=="rideH",vehicle_type:="TNC"] - ev[vehicle_type=="subway",vehicle_type:="BART"] + ev[vehicle_type=="bus",vehicle_type:="BUS-DEFAULT"] + ev[vehicle_type=="CAR",vehicle_type:="Car"] + ev[substr(vehicle,1,5)=="rideH",mode:="ride_hail"] + ev[vehicle_type=="subway",vehicle_type:="SUBWAY-DEFAULT"] ev[vehicle_type=="SUV",vehicle_type:="Car"] - ev[vehicle_type=="cable_car",vehicle_type:="Cable_Car"] - ev[vehicle_type=="tram",vehicle_type:="Muni"] - ev[vehicle_type=="rail",vehicle_type:="Rail"] - ev[vehicle_type=="ferry",vehicle_type:="Ferry"] + ev[vehicle_type=="cable_car",vehicle_type:="CABLE_CAR-DEFAULT"] + ev[vehicle_type=="tram",vehicle_type:="TRAM-DEFAULT"] + ev[vehicle_type=="rail",vehicle_type:="RAIL-DEFAULT"] + ev[vehicle_type=="ferry",vehicle_type:="FERRY-DEFAULT"] + transit.types <- c('BUS-DEFAULT','FERRY-DEFAULT','TRAM-DEFAULT','RAIL-DEFAULT','CABLE_CAR-DEFAULT','SUBWAY-DEFAULT') ev[,tripmode:=ifelse(mode%in%c('subway','bus','rail','tram','walk_transit','drive_transit','cable_car','ferry'),'transit',as.character(mode))] ev[,hour:=time/3600] ev[,hr:=round(hour)] @@ -25,9 +27,9 @@ clean.and.relabel <- function(ev,factor.to.scale.personal.back,factor.to.scale.t if(is.factor(ev$fuel[1]))ev[,fuel:=as.numeric(as.character(fuel))] ev[start.y<=0.003 | end.y <=0.003,':='(start.x=NA,start.y=NA,end.x=NA,end.y=NA)] ev[length==Inf,length:=NA] - ev[vehicle_type%in%c('BART','Ferry','Muni','Rail','Cable_Car') & !is.na(start.x) & !is.na(start.y) & !is.na(end.y) & !is.na(end.y),length:=dist.from.latlon(start.y,start.x,end.y,end.x)] - ev[vehicle_type%in%c('BART','Bus','Cable_Car','Muni','Rail'),num_passengers:=round(num_passengers*factor.to.scale.personal.back)] - ev[vehicle_type%in%c('BART','Bus','Cable_Car','Muni','Rail'),capacity:=round(capacity*factor.to.scale.transit.back)] + ev[vehicle_type%in%transit.types & !is.na(start.x) & !is.na(start.y) & !is.na(end.y) & !is.na(end.y),length:=dist.from.latlon(start.y,start.x,end.y,end.x)] + ev[vehicle_type%in%transit.types,num_passengers:=round(num_passengers*factor.to.scale.personal.back)] + ev[vehicle_type%in%transit.types,capacity:=round(capacity*factor.to.scale.transit.back)] ev[num_passengers > capacity,num_passengers:=capacity] ev[,pmt:=num_passengers*length/1609] ev[is.na(pmt),pmt:=0] @@ -41,8 +43,8 @@ clean.and.relabel <- function(ev,factor.to.scale.personal.back,factor.to.scale.t ev } -pretty.titles <- c('TNC Number'='ridehail_num', - 'TNC Price'='ridehail_price', +pretty.titles <- c('Ride Hail Number'='ridehail_num', + 'Ride Hail Price'='ridehail_price', 'Transit Capacity'='transit_capacity', 'Transit Price'='transit_price', 'Toll Price'='toll_price', @@ -57,8 +59,8 @@ to.title <- function(abbrev){ } } pretty.modes <- function(ugly){ - pretty.list <- c('TNC'='ride_hail', - 'TNC'='ride_hail', + pretty.list <- c('Ride Hail'='ride_hail', + 'Ride Hail - Transit'='ride_hail_transit', 'Cable Car'='cable_car', 'Car'='car', 'Walk'='walk', @@ -89,7 +91,7 @@ parse.link.stats <- function(link.stats.file,net.file=NA){ } my.colors <- c(blue='#377eb8',green='#227222',orange='#C66200',purple='#470467',red='#B30C0C',yellow='#C6A600',light.green='#C0E0C0',magenta='#D0339D',dark.blue='#23128F',brown='#542D06',grey='#8A8A8A',dark.grey='#2D2D2D',light.yellow='#FFE664',light.purple='#9C50C0',light.orange='#FFB164',black='#000000') -mode.colors <- c(TNC='red',Car='grey',Walk='green',Transit='blue') +mode.colors <- c('Ride Hail'='red',Car='grey',Walk='green',Transit='blue','Ride Hail - Transit'='purple') mode.colors <- data.frame(key=names(mode.colors),color=mode.colors,color.hex=my.colors[mode.colors]) download.from.nersc <- function(experiment.dir,include.pattern='*'){ diff --git a/src/main/R/bin/csv2Rdata.R b/src/main/R/bin/csv2Rdata.R index 2cb70af5350..58a91561281 100755 --- a/src/main/R/bin/csv2Rdata.R +++ b/src/main/R/bin/csv2Rdata.R @@ -15,8 +15,7 @@ option_list <- list( ) if(interactive()){ #setwd('~/downs/') - args<-'/Users/critter/Downloads/2014-Clipper.csv' - args<-'/Users/critter/Downloads/output 3/application-sfbay/base__2018-06-18_16-21-35/ITERS/it.1/1.events.csv' + args<-'/Users/critter/Downloads/output 2/application-sfbay/RH2Transit_Plus10__2018-09-26_04-55-53/ITERS/it.10/10.events.csv' args <- parse_args(OptionParser(option_list = option_list,usage = "csv2Rdata.R [file-to-convert]"),positional_arguments=T,args=args) }else{ args <- parse_args(OptionParser(option_list = option_list,usage = "csv2Rdata.R [file-to-convert]"),positional_arguments=T) diff --git a/src/main/R/bin/exp2plots.R b/src/main/R/bin/exp2plots.R index 7a257f4dd4b..0433504f9e3 100755 --- a/src/main/R/bin/exp2plots.R +++ b/src/main/R/bin/exp2plots.R @@ -17,17 +17,20 @@ option_list <- list( ) if(interactive()){ #setwd('~/downs/') - args<-'/Users/critter/Documents/beam/beam-output/experiments/2018-04/surge-pricing/' - args<-'/Users/critter/Documents/beam/beam-output/experiments/2018-04/ridehail-price/' + args<-'/Users/critter/Downloads/RH2Transit-2Iter/RH2Transit' + args<-'/Users/critter/Downloads/cost-sensitivities/cost-sensitivity/' + #args<-'/Users/critter/Downloads/diffusion-5iter/diffusion/' args <- parse_args(OptionParser(option_list = option_list,usage = "exp2plots.R [experiment-directory]"),positional_arguments=T,args=args) }else{ args <- parse_args(OptionParser(option_list = option_list,usage = "exp2plots.R [experiment-directory]"),positional_arguments=T) } ###################################################################################################### -factor.to.scale.personal.back <- 35 -factor.to.scale.transit.back <- 2 +# TODO make these come from conf file +factor.to.scale.personal.back <- 20 +factor.to.scale.transit.back <- 1/.22 # inverse of param: beam.agentsim.tuning.transitCapacity plot.congestion <- F +plot.modality.styles <- F ###################################################################################################### # Load the exp config @@ -72,21 +75,25 @@ for(run.i in 1:nrow(exp)){ } links[[length(links)+1]] <- link } - last.iter.minus.5 <- ifelse(last.iter>4,last.iter - 4,1) - for(the.iter in last.iter.minus.5:last.iter){ - pop.csv <- pp(run.dir,output.dir,'/ITERS/it.',the.iter,'/',the.iter,'.population.csv.gz') - if(file.exists(pop.csv)){ - pop <- csv2rdata(pop.csv) - pop[,iter:=the.iter] - streval(pp('pop[,',fact,':="',the.level,'"]')) - pops[[length(pops)+1]] <- pop + if(plot.modality.styles){ + last.iter.minus.5 <- ifelse(last.iter>4,last.iter - 4,1) + for(the.iter in last.iter.minus.5:last.iter){ + pop.csv <- pp(run.dir,output.dir,'/ITERS/it.',the.iter,'/',the.iter,'.population.csv.gz') + if(file.exists(pop.csv)){ + pop <- csv2rdata(pop.csv) + pop[,iter:=the.iter] + streval(pp('pop[,',fact,':="',the.level,'"]')) + pops[[length(pops)+1]] <- pop + } } } } ev <- rbindlist(evs,use.names=T,fill=T) rm('evs') -pop <- rbindlist(pops,use.names=T,fill=T) -rm('pops') +if(plot.modality.styles){ + pop <- rbindlist(pops,use.names=T,fill=T) + rm('pops') +} if(plot.congestion){ link <- rbindlist(links,use.names=T,fill=T) rm('links') @@ -97,12 +104,12 @@ ev <- clean.and.relabel(ev,factor.to.scale.personal.back,factor.to.scale.transit setkey(ev,type) ## Prep data needed to do quick version of energy calc -en <- data.table(read.csv('~/Dropbox/ucb/vto/beam-all/beam/test/input/sf-light/energy/energy-consumption.csv')) -setkey(en,vehicleType) +en <- data.table(read.csv('~/Dropbox/ucb/vto/beam-all/beam/production/application-sfbay/samples/vehicleTypes.csv')) +setkey(en,vehicleTypeId) en <- u(en) ## Energy Density in MJ/liter or MJ/kWh # https://en.wikipedia.org/wiki/Gasoline_gallon_equivalent -en.density <- data.table(fuelType=c('gasoline','diesel','electricity'),density=c(31.81905,36.14286,3.6)) +#en.density <- data.table(fuelType=c('gasoline','diesel','electricity','biodiesel'),density=c(31.81905,36.14286,3.6,33.2)) ev[tripmode%in%c('car') & vehicle_type=='Car',':='(num_passengers=1)] ev[,pmt:=num_passengers*length/1609] ev[is.na(pmt),pmt:=0] @@ -132,8 +139,9 @@ for(fact in factors){ toplot[,frac:=num/tot] toplot[,tripmode:=pretty.modes(tripmode)] setkey(toplot,the.factor,tripmode) - p <- ggplot(toplot,aes(x=the.factor,y=frac*100,fill=tripmode))+geom_bar(stat='identity',position='stack')+labs(x="Scenario",y="% of Trips",title=pp('Factor: ',fact),fill="Trip Mode")+ - scale_fill_manual(values=as.character(mode.colors$color.hex[match(sort(u(toplot$tripmode)),mode.colors$key)])) + p <- ggplot(toplot,aes(x=the.factor,y=frac*100,fill=tripmode))+geom_bar(stat='identity',position='stack')+ + labs(x="Scenario",y="% of Trips",title=pp('Factor: ',fact),fill="Trip Mode")+ + scale_fill_manual(values=as.character(mode.colors$color.hex[match(sort(u(toplot$tripmode)),mode.colors$key)])) pdf.scale <- .6 ggsave(pp(plots.dir,'mode-split-by-',fact,'.pdf'),p,width=10*pdf.scale,height=6*pdf.scale,units='in') write.csv(toplot,file=pp(plots.dir,'mode-split-by-',fact,'.csv')) @@ -150,7 +158,7 @@ for(fact in factors){ ggsave(pp(plots.dir,'mode-split-by-hour-by-',fact,'.pdf'),p,width=10*pdf.scale,height=6*pdf.scale,units='in') write.csv(toplot,file=pp(plots.dir,'mode-split-by-hour-',fact,'.csv')) - target <- data.frame(tripmode=rep(c('Car','Walk','Transit','TNC'),length(u(toplot$the.factor))), + target <- data.frame(tripmode=rep(c('Car','Walk','Transit','Ride Hail'),length(u(toplot$the.factor))), perc=rep(c(79,4,13,5),length(u(toplot$the.factor))), the.factor=rep(u(toplot$the.factor),each=4)) p <- ggplot(toplot,aes(x=tripmode,y=frac*100))+geom_bar(stat='identity')+facet_wrap(~the.factor)+geom_point(data=target,aes(y=perc),colour='red') @@ -188,24 +196,23 @@ for(fact in factors){ toplot <- pt[,.(fuel=sum(fuel),numVehicles=as.double(length(fuel)),numberOfPassengers=as.double(sum(num_passengers)),pmt=sum(pmt),vmt=sum(length)/1609,mpg=sum(length)/1609/sum(fuel/3.78)),by=c('the.factor','vehicle_type','tripmode')] toplot <- toplot[vehicle_type!='Human' & tripmode!="walk"] if(nrow(en)>30){ - toplot <- join.on(toplot,en[vehicle%in%c('SUBWAY-DEFAULT','BUS-DEFAULT','CABLE_CAR-DEFAULT','CAR','FERRY-DEFAULT','TRAM-DEFAULT','RAIL-DEFAULT')|vehicleType=='TNC'],'vehicle_type','vehicleType','fuelType') + toplot <- join.on(toplot,en[vehicleTypeId%in%c('SUBWAY-DEFAULT','BUS-DEFAULT','CABLE_CAR-DEFAULT','Car','FERRY-DEFAULT','TRAM-DEFAULT','RAIL-DEFAULT') | substr(vehicleTypeId,0,3)=='BEV'],'vehicle_type','vehicleTypeId','primaryFuelType') }else{ - toplot <- join.on(toplot,en,'vehicle_type','vehicleType','fuelType') + toplot <- join.on(toplot,en,'vehicle_type','vehicleTypeId','primaryFuelType') } - toplot <- join.on(toplot,en.density,'fuelType','fuelType') - toplot[,energy:=fuel*density] - toplot[vehicle_type=='TNC',tripmode:='TNC'] - toplot[vehicle_type%in%c('Car','TNC'),energy:=energy*factor.to.scale.personal.back*10] - toplot[vehicle_type%in%c('Car','TNC'),numVehicles:=numVehicles*factor.to.scale.personal.back] - toplot[vehicle_type%in%c('Car','TNC'),pmt:=pmt*factor.to.scale.personal.back] - toplot[vehicle_type%in%c('Car','TNC'),numberOfPassengers:=numVehicles] + toplot[,energy:=fuel] # fuel is now in units of J so no need to convert from L to J + toplot[tripmode%in%c('car','ride_hail'),energy:=energy*factor.to.scale.personal.back] + toplot[tripmode%in%c('car','ride_hail'),numVehicles:=numVehicles*factor.to.scale.personal.back] + toplot[tripmode%in%c('car','ride_hail'),pmt:=pmt*factor.to.scale.personal.back] + toplot[tripmode%in%c('car','ride_hail'),numberOfPassengers:=numVehicles] toplot[,ag.mode:=tripmode] toplot[tolower(ag.mode)%in%c('bart','bus','cable_car','muni','rail','tram','transit'),ag.mode:='Transit'] toplot[ag.mode=='car',ag.mode:='Car'] + toplot[ag.mode=='ride_hail',ag.mode:='Ride Hail'] toplot.ag <- toplot[,.(energy=sum(energy),pmt=sum(pmt)),by=c('the.factor','ag.mode')] pdf.scale <- .6 setkey(toplot.ag,the.factor,ag.mode) - p <- ggplot(toplot.ag,aes(x=the.factor,y=energy/1e6,fill=ag.mode))+geom_bar(stat='identity',position='stack')+labs(x="Scenario",y="Energy Consumption (TJ)",title=to.title(fact),fill="Trip Mode")+ + p <- ggplot(toplot.ag,aes(x=the.factor,y=energy/1e12,fill=ag.mode))+geom_bar(stat='identity',position='stack')+labs(x="Scenario",y="Energy Consumption (TJ)",title=to.title(fact),fill="Trip Mode")+ scale_fill_manual(values=as.character(mode.colors$color.hex[match(sort(u(toplot.ag$ag.mode)),mode.colors$key)])) ggsave(pp(plots.dir,'energy-by-mode-',fact,'.pdf'),p,width=10*pdf.scale,height=6*pdf.scale,units='in') write.csv(toplot.ag,file=pp(plots.dir,'energy-by-mode-',fact,'.csv')) @@ -219,11 +226,11 @@ for(fact in factors){ ggsave(pp(plots.dir,'energy-per-pmt-by-vehicle-type-',fact,'.pdf'),p,width=10*pdf.scale,height=6*pdf.scale,units='in') # Deadheading - toplot <- pt[vehicle_type=='TNC',.(dead=num_passengers==0,miles=length/1609,hr,the.factor)] + toplot <- pt[tripmode=='ride_hail',.(dead=num_passengers==0,miles=length/1609,hr,the.factor)] setkey(toplot,hr,dead) dead.frac <- toplot[,.(dead.frac=pp(roundC(100*sum(miles[dead==T])/sum(miles),1),"% Empty")),by=c('the.factor')] toplot <- toplot[,.(miles=sum(miles)),by=c('dead','hr','the.factor')] - p <- ggplot(toplot,aes(x=hr,y=miles,fill=dead))+geom_bar(stat='identity')+labs(x="Hour",y="Vehicle Miles Traveled",fill="Empty",title=pp("TNC Deadheading"))+geom_text(data=dead.frac,hjust=1,aes(x=24,y=max(toplot$miles),label=dead.frac,fill=NA))+facet_wrap(~the.factor) + p <- ggplot(toplot,aes(x=hr,y=miles,fill=dead))+geom_bar(stat='identity')+labs(x="Hour",y="Vehicle Miles Traveled",fill="Empty",title=pp("Ride Hail Deadheading"))+geom_text(data=dead.frac,hjust=1,aes(x=24,y=max(toplot$miles),label=dead.frac,fill=NA))+facet_wrap(~the.factor) pdf.scale <- .6 ggsave(pp(plots.dir,'dead-heading.pdf'),p,width=10*pdf.scale,height=6*pdf.scale,units='in') } @@ -244,32 +251,34 @@ if(plot.congestion){ } } -if('customAttributes' %in% names(pop)){ - for(fact in factors){ - streval(pp('pop[,the.factor:=',fact,']')) - if(all(c('low','base','high') %in% u(pop$the.factor))){ - pop[,the.factor:=factor(the.factor,levels=c('low','base','high'))] - }else if(all(c('Low','Base','High') %in% u(pop$the.factor))){ - pop[,the.factor:=factor(the.factor,levels=c('Low','Base','High'))] - }else{ - streval(pp('pop[,the.factor:=factor(the.factor,levels=exp$',fact,')]')) - } - pop[,style:=customAttributes] - pop <- pop[style!=''] - if(any(u(pop$style)=='class1')){ - new.names <- c(class1='Multimodals',class2='Empty nesters',class3='Transit takers',class4='Inveterate drivers',class5='Moms in cars',class6='Car commuters') - pop[,style:=new.names[as.character(style)]] - } - toplot <- pop[,.(n=length(type)),by=c('style','the.factor','iter')] - toplot <- toplot[,.(n=mean(n)),by=c('style','the.factor')] - setkey(toplot,style,the.factor) +if(plot.modality.styles){ + if('customAttributes' %in% names(pop)){ + for(fact in factors){ + streval(pp('pop[,the.factor:=',fact,']')) + if(all(c('low','base','high') %in% u(pop$the.factor))){ + pop[,the.factor:=factor(the.factor,levels=c('low','base','high'))] + }else if(all(c('Low','Base','High') %in% u(pop$the.factor))){ + pop[,the.factor:=factor(the.factor,levels=c('Low','Base','High'))] + }else{ + streval(pp('pop[,the.factor:=factor(the.factor,levels=exp$',fact,')]')) + } + pop[,style:=customAttributes] + pop <- pop[style!=''] + if(any(u(pop$style)=='class1')){ + new.names <- c(class1='Multimodals',class2='Empty nesters',class3='Transit takers',class4='Inveterate drivers',class5='Moms in cars',class6='Car commuters') + pop[,style:=new.names[as.character(style)]] + } + toplot <- pop[,.(n=length(type)),by=c('style','the.factor','iter')] + toplot <- toplot[,.(n=mean(n)),by=c('style','the.factor')] + setkey(toplot,style,the.factor) - pdf.scale <- .8 - p<-ggplot(toplot,aes(x=the.factor,y=n,fill=style))+geom_bar(stat='identity',position='stack')+labs(x='Trip #',y='# Agents',title='Modality Style Distributions') - ggsave(pp(plots.dir,'modality-styles.pdf'),p,width=10*pdf.scale,height=8*pdf.scale,units='in') + pdf.scale <- .8 + p<-ggplot(toplot,aes(x=the.factor,y=n,fill=style))+geom_bar(stat='identity',position='stack')+labs(x='Trip #',y='# Agents',title='Modality Style Distributions') + ggsave(pp(plots.dir,'modality-styles.pdf'),p,width=10*pdf.scale,height=8*pdf.scale,units='in') - p<-ggplot(toplot,aes(x=style,y=n,fill=the.factor))+geom_bar(stat='identity',position='dodge')+labs(x='',y='# Agents',fill=fact,title='Modality Style Distributions') - ggsave(pp(plots.dir,'modality-styles-dodged.pdf'),p,width=10*pdf.scale,height=6*pdf.scale,units='in') + p<-ggplot(toplot,aes(x=style,y=n,fill=the.factor))+geom_bar(stat='identity',position='dodge')+labs(x='',y='# Agents',fill=fact,title='Modality Style Distributions') + ggsave(pp(plots.dir,'modality-styles-dodged.pdf'),p,width=10*pdf.scale,height=6*pdf.scale,units='in') + } } } diff --git a/src/main/R/bin/exp2ridehail.R b/src/main/R/bin/exp2ridehail.R index 0026909bab2..37083c149e1 100755 --- a/src/main/R/bin/exp2ridehail.R +++ b/src/main/R/bin/exp2ridehail.R @@ -119,9 +119,9 @@ rh.sum <- rh[,.(n.vehicle.legs=.N, avg.trip.len.with.passenger=sum(length[num_passengers>0])/sum(num_passengers>0)/1608, avg.trip.deadhead.miles=sum(length[num_passengers==0 & reposition==F])/sum(num_passengers==0 & reposition==F)/1608, avg.trip.reposition.miles=sum(length[num_passengers==0 & reposition==T])/sum(num_passengers==0 & reposition==T)/1608, - empty.vmt.fraction=sum(length[num_passengers==0])/sum(length)/1608, - deadhead.vmt.fraction=sum(length[num_passengers==0& reposition==F])/sum(length)/1608, - reposition.vmt.fraction=sum(length[num_passengers==0& reposition==T])/sum(length)/1608, + empty.vmt.fraction=sum(length[num_passengers==0])/sum(length), + deadhead.vmt.fraction=sum(length[num_passengers==0& reposition==F])/sum(length), + reposition.vmt.fraction=sum(length[num_passengers==0& reposition==T])/sum(length), avg.speed=mean(speed), avg.speed.vmt.weighted=weighted.mean(speed,length) ),by='run'] @@ -209,4 +209,84 @@ if(F){ #ggplot(poprh,aes(x=x,y=y,colour=type))+geom_point(alpha=.2) +park.dir <- '~/Dropbox/ucb/vto/beam-all/beam/production/application-sfbay/parking/' + +chs <- list() +dir <- tail(list.dirs(park.dir),-1)[1] +for(dir in tail(list.dirs(park.dir),-1)){ + for(ch.file in list.files(dir)){ + ch <- data.table(read.csv(pp(dir,'/',ch.file))) + ch[,file:=ch.file] + chs[[length(chs)+1]]<-ch + } +} +ch <- rbindlist(chs) + +ch[,charge:=ifelse(grepl('P250',file),'P250','P50')] +ch[,range:=ifelse(grepl('R150',file),'R150','R75')] +ch[,qos:=ifelse(grepl('S60',file),'S60',ifelse(grepl('S70',file),'S70',ifelse(grepl('S80',file),'S80',ifelse(grepl('S90',file),'S90',ifelse(grepl('S95',file),'S95','S98')))))] +ch.sum <- ch[reservedFor=='RideHailManager' & numStalls<2147483000 & chargingType=='DCFast',.(nchargers=sum(numStalls)),by=c('charge','range','qos')] + +library(stringr) + +save(all,exp,file=pp(plots.dir,'/summary-metrics-all.Rdata')) +load(file=pp(plots.dir,'/summary-metrics-all.Rdata')) + + +load("/Users/critter/Documents/beam/beam-output/EVFleet-Final/EVFleet-2018-09-24/plots/summary-metrics.Rdata",verb=T) +load("/Users/critter/Documents/beam/beam-output/EVFleet-Final/EVFleet-2018-09-24/plots-all/summary-metrics-all.Rdata",verb=T) + +# adding base to all +all <- rbindlist(list(all,all.sum),fill=T,use.names=T) +all[chargers=='base',range:='Base'] +all[chargers=='base',charge:='N/A'] +all[chargers=='base',qos:='N/A'] + +all <- all[!(range=='R150' & qos=='S98' & charge=='P250')] +all <- join.on(all,ch.sum,c('charge','range','qos'),c('charge','range','qos'),'nchargers') +all[chargers=='base',nchargers:=0] + +all[range!='Base',empty.vmt.fraction:=empty.vmt.fraction*1608] +all[range!='Base',deadhead.vmt.fraction:=empty.vmt.fraction*1608] + +all.m <- melt(all,id.vars=c('range','charge','qos','nchargers')) +all.m[,variable:=str_replace_all(variable,'\\.','_')] +all.m[variable=='empty_vmt_fraction',value:=as.character(as.numeric(value)*100)] +all.m[variable=='empty_vmt_fraction' & range=='Base',value:=as.character(as.numeric(value)/1608)] + + +#for(metric in u(all.m$variable)){ + #if(all(is.na(as.numeric(all.m[variable==metric]$value)))){ + ## do nothing + #}else{ + #toplot <- all.m[variable==metric & qos!='S70'] + #toplot[,value:=as.numeric(value)] + #p<-ggplot(toplot,aes(x=qos,y= value))+geom_bar(stat='identity')+facet_grid(range~charge)+labs(title=metric) + #pdf.scale <- 0.75 + #ggsave(pp(plots.dir,'metric-',metric,'.pdf'),p,width=11*pdf.scale,height=6*pdf.scale,units='in') + #} +#} + +all.m[charge=='P50' & qos=='S80',qos:='S99'] +all.m[charge=='P50' & qos=='S99',charge:='P250'] +all.m[,qosn:=as.numeric(substr(qos,2,4))] + +#ggplot(all.m[charge=='P50'&variable%in%c('vmt_per_vehicle','n_trips_with_passenger','n_charge_sessions','customer_wait_50th_percentile')],aes(x=qosn,y=as.numeric(value),colour=range))+geom_line()+facet_wrap(~variable) + +ggplot(all.m[qos!='S70'&charge=='P250'&variable%in%c('customer_wait_99th_percentile','n_trips_with_passenger','n_charge_sessions','customer_wait_50th_percentile')],aes(x=nchargers,y=as.numeric(value),colour=range,shape=range))+ + geom_line()+ + geom_point()+ + geom_hline(data=all.m[range=='Base'&variable%in%c('customer_wait_99th_percentile','n_trips_with_passenger','n_charge_sessions','customer_wait_50th_percentile')],aes(yintercept=as.numeric(value)),color=my.colors['red'])+ + scale_colour_manual(values=as.vector(my.colors[c('blue','green')]))+ + facet_wrap(~variable,scales='free_y') + +dev.new() + +ggplot(all.m[qos!='S70'&charge=='P250'&variable%in%c('vmt_per_vehicle','total_vmt','empty_vmt_fraction','avg_trip_deadhead_miles')],aes(x=nchargers,y=as.numeric(value),colour=range,shape=range))+ + geom_line()+ + geom_point()+ + geom_hline(data=all.m[range=='Base'&variable%in%c('vmt_per_vehicle','total_vmt','empty_vmt_fraction','avg_trip_deadhead_miles')],aes(yintercept=as.numeric(value)),color=my.colors['red'])+ + scale_colour_manual(values=as.vector(my.colors[c('blue','green')]))+ + facet_wrap(~variable,scales='free_y') + diff --git a/src/main/R/bin/indiv2exp.R b/src/main/R/bin/indiv2exp.R index 0a6f53f0f21..9a9e5e9be2f 100755 --- a/src/main/R/bin/indiv2exp.R +++ b/src/main/R/bin/indiv2exp.R @@ -19,8 +19,8 @@ option_list <- list( ) if(interactive()){ #setwd('~/downs/') - args<-c('pruning-1','/Users/critter/Documents/beam/beam-output/experiments/base_2018-01-19_21-23-36 2/','/Users/critter/Documents/beam/beam-output/experiments/pruned-transit_2018-01-19_06-13-51/') - #args<-'/Users/critter/Documents/beam/beam-output/experiments/prices-25k/' + #args<-c('pruning-1','/Users/critter/Documents/beam/beam-output/experiments/base_2018-01-19_21-23-36 2/','/Users/critter/Documents/beam/beam-output/experiments/pruned-transit_2018-01-19_06-13-51/') + args<-c('rh2transit','/Users/critter/Downloads/output 2/application-sfbay/RH2Transit_Plus10__2018-09-26_04-55-53/') args <- parse_args(OptionParser(option_list = option_list,usage = "indiv2exp.R [exp-name] [run-directories]"),positional_arguments=T,args=args) }else{ args <- parse_args(OptionParser(option_list = option_list,usage = "indiv2exp.R [exp-name] [run-directories]"),positional_arguments=T) diff --git a/src/main/R/pems-to-network.R b/src/main/R/pems-to-network.R new file mode 100644 index 00000000000..50e27da1488 --- /dev/null +++ b/src/main/R/pems-to-network.R @@ -0,0 +1,78 @@ + +load.libraries(c('optparse','XML','stringr','rgdal','sp'),quietly=T) + +load("/Users/critter/Dropbox/ucb/vto/beam-all/beam/production/application-sfbay/physsim-network.Rdata") +pems <- data.table(read.csv('~/Dropbox/ucb/vto/beam-all/beam/production/application-sfbay/counts/preprocessing/matched_sensors.csv')) + +linknodes <- join.on(join.on(links,nodes,'from','id',c('x','y'),'from.'),nodes,'to','id',c('x','y'),'to.') +linknodes[,cent.x:=(from.x+to.x)/2] +linknodes[,cent.y:=(from.y+to.y)/2] +reproj.nodes <- linknodes +coordinates(reproj.nodes) <- ~ cent.x+cent.y +proj4string(reproj.nodes) <- CRS("+init=epsg:26910") +reproj.nodes.wgs <- spTransform(reproj.nodes,CRS("+init=epsg:4326")) +linknodes[,cent.lon:=coordinates(reproj.nodes.wgs)[,1]] +linknodes[,cent.lat:=coordinates(reproj.nodes.wgs)[,2]] + + +thresh <- 0.005 +for(i in 1:nrow(pems)){ + if(i%%25==0)my.cat(i) + search.set <- linknodes[abs(cent.lon - pems[i]$Longitude)\n +\n' +cat(the.str,file=outfile,append=F) +i <- inds[1] +for(i in inds){ + id <- pems$ID[i] + link <- pems$link[i] + the.str <- pp('\t\n') + the.cnts <- pp('\t\t\n') + the.close <- pp('\t\n') + cat(the.str,file=outfile,append=T) + cat(the.cnts,file=outfile,append=T) + cat(the.close,file=outfile,append=T) +} +the.str <- '' +cat(the.str,file=outfile,append=T) + diff --git a/src/main/R/population.R b/src/main/R/population.R new file mode 100644 index 00000000000..9bd4098c7e8 --- /dev/null +++ b/src/main/R/population.R @@ -0,0 +1,50 @@ + +load.libraries(c('stringr')) + +samp.dir <- '/Users/critter/Dropbox/ucb/vto/beam-all/beam/production/application-sfbay/samples' + +hh <- data.table(read.csv(pp(samp.dir,'/hh-members.csv'),header=F,stringsAsFactors=F)) +hh[,ndash:=str_count(V1,'-')] + +hh.inc <- hh[ndash==0] +hh.inc[,income:=as.numeric(substr(as.character(V1),3,nchar(as.character(V1))))] +hh.inc[,V1:=NULL] +hh.veh <- hh[ndash==1] +hh.id <- hh[ndash==2] +hh.id[,income:=hh.inc$income] +hh <- hh.id + +hh2 <- data.table(read.csv(pp(samp.dir,'/hh-id.csv'),header=F)) +hh2[,x:=data.table(read.csv(pp(samp.dir,'/hh-x.csv'),header=F))$V1] +hh2[,y:=data.table(read.csv(pp(samp.dir,'/hh-y.csv'),header=F))$V1] + +hh <- join.on(hh,hh2,'V1','V1') +hh[,':='(id=V1,V1=NULL)] +hh[,':='(ndash=NULL)] + +vot <- data.table(read.csv(pp(samp.dir,'/vott-id.csv'),header=F)) +vot[,vot:=data.table(read.csv(pp(samp.dir,'/vott-value.csv'),header=F))$V1] +vot[,':='(id=V1,V1=NULL)] + +load("/Users/critter/Dropbox/ucb/vto/beam-all/beam/production/application-sfbay/samples/population.Rdata",verb=T) +pop <- copy(df) +rm('df') +pop <- join.on(pop,vot,'id','id') +if(!'hh'%in%names(pop)){ + pop[,hh:=unlist(lapply(str_split(id,'-'),function(ll){ pp(head(ll,-1),collapse='-') }))] +} +pop <- join.on(pop,hh,'hh','id',included.prefix='hh.') +save(pop,hh,file="/Users/critter/Dropbox/ucb/vto/beam-all/beam/production/application-sfbay/samples/pop-full.Rdata",verb=T) + +setkey(pop,id) +pop <- u(pop) +pop[hh.income<1000,hh.income:=1000] +pop[vot<1,vot:=1] +pop[,income.bin.10k:=floor(hh.income/10000)*10] +pop[income.bin.10k>=100 & income.bin.10k<150,income.bin.10k:=100] +pop[income.bin.10k>=150 & income.bin.10k<200,income.bin.10k:=150] +pop[income.bin.10k>=200,income.bin.10k:=200] + +ggplot(pop,aes(x= factor(income.bin.10k),y=vot))+ geom_boxplot()+labs(x="Annual Household Income ($1000)",y="Value of Travel Time ($/hr)",title="SF Bay Area Population Value of Travel Time") +dev.new() +ggplot(pop,aes(x= hh.income/1000))+ geom_histogram()+labs(x="Annual Household Income ($1000)",y="Frequency",title="SF Bay Area Population Household Income") diff --git a/src/main/R/viz-plans.R b/src/main/R/viz-plans.R new file mode 100644 index 00000000000..1dac34f4943 --- /dev/null +++ b/src/main/R/viz-plans.R @@ -0,0 +1,32 @@ + +load("/Users/critter/Dropbox/ucb/vto/beam-all/beam/production/application-sfbay/samples/population.Rdata") + +reproj(df,'epsg:26910','epsg:4326')->newdt + +newdt[,time:=ifelse(end.time>=0,end.time,tail(end.time,2)[1]+10*3600),by='id'] + + +write.csv(newdt[,.(x=new.x,y=new.y,time,type)],file='/Users/critter/Dropbox/ucb/vto/beam-colin/viz-2018/activities.csv') + +popsamp <- sample(newdt$id,100000) +write.csv(newdt[id%in%popsamp,.(x=new.x,y=new.y,time,type)],file='/Users/critter/Dropbox/ucb/vto/beam-colin/viz-2018/activities-sm.csv') + + +# Now viz movements + +# load an events file + +pt <- df[type=='PathTraversal' & length>0] +pt[substr(vehicle,0,5)=='rideH',mode:='ride_hail'] +setkey(pt,mode) + +write.csv(pt[,.(mode,start.x,start.y,end.x,end.y, time=departure_time,num= num_passengers)],file=pp('/Users/critter/Dropbox/ucb/vto/beam-colin/viz-2018/pt.csv'),row.names=F) + +for(the.mode in u(pt$mode)){ + write.csv(pt[J(the.mode),.(start.x,start.y,end.x,end.y, time=departure_time+.01,num= num_passengers+0.1)],file=pp('/Users/critter/Dropbox/ucb/vto/beam-colin/viz-2018/pt-',the.mode,'.csv'),row.names=F) +} + +# do walk without bushwacking +the.mode<-'walk' +write.csv(pt[J(the.mode)][links!="",.(start.x,start.y,end.x,end.y, time=departure_time+.01,num= num_passengers+0.1)],file=pp('/Users/critter/Dropbox/ucb/vto/beam-colin/viz-2018/pt-',the.mode,'.csv'),row.names=F) + diff --git a/src/main/bash/copyAllFilesFromRunningExperiment.sh b/src/main/bash/copyAllFilesFromRunningExperiment.sh index ad63c8bdb8e..6f6ec72ce2b 100644 --- a/src/main/bash/copyAllFilesFromRunningExperiment.sh +++ b/src/main/bash/copyAllFilesFromRunningExperiment.sh @@ -1,3 +1,4 @@ +#!/usr/bin/env bash sudo chmod 600 ~/.ssh/result_host_cert.pem @@ -61,6 +62,9 @@ else sudo mkdir "$1/copied" + + + echo "sourceExperimentDir: $sourceExperimentDir" echo "copiedExperiementDir: $copiedExperiementDir" @@ -68,10 +72,42 @@ else echo "Empty source directory $sourceExperimentDir" exit 0 else + sourceExperimentZipped="${exp_id}_${dir_date}.tar.gz" + + echo "changing dir .." + cd "$1/to_copy" + pwd + + +# zipcmd="sudo tar -zcf ${sourceExperimentZipped} ${exp_id}" +# echo "Running command: $zipcmd" +# $zipcmd + +# echo "Zipping completed - " +# du -sh ${sourceExperimentZipped} + + echo "Copying the files... from $sourceExperimentDir" sudo scp -i ~/.ssh/result_host_cert.pem -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -r $sourceExperimentDir ubuntu@$2:~/sigoptResults/ echo "Copying completed..." + +# echo "Unzipping on the result server" +# +# sudo ssh -i ~/.ssh/result_host_cert.pem -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null ubuntu@$2: cd sigoptResults +# sudo ssh -i ~/.ssh/result_host_cert.pem -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null ubuntu@$2: du -sh ${sourceExperimentZipped} +# +# unzipcmd="sudo tar -zxf ${sourceExperimentZipped}" +# sudo ssh -i ~/.ssh/result_host_cert.pem -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null ubuntu@$2: $unzipcmd +# echo "Unzipping completed" + + +# echo "Running the command mv $sourceExperimentZipped $copiedExperiementDir" +# sudo mv "$sourceExperimentZipped" "$copiedExperiementDir" + + echo "Running the command mv $sourceExperimentDir $copiedExperiementDir" sudo mv "$sourceExperimentDir" "$copiedExperiementDir" + + echo "Moved the suggestions from $sourceExperimentDir to $copiedExperiementDir" echo "Done.." fi diff --git a/src/main/bash/deploy-examples.sh b/src/main/bash/deploy-examples.sh new file mode 100644 index 00000000000..65d54636bb1 --- /dev/null +++ b/src/main/bash/deploy-examples.sh @@ -0,0 +1,29 @@ + +gradle deploy -PbeamConfigs=production/application-sfbay/diffusion-and-rh2transit-base.conf,production/application-sfbay/diffusion-ridehail.conf,production/application-sfbay/diffusion-av.conf -PbeamCommit=2ce2248ef3f2659c12ff6788b01eee1f0f806b43 -Pbatch=false -PrunName=diffusionExperiment + +gradle deploy -PbeamConfigs=production/application-sfbay/rh2transit-max.conf,production/application-sfbay/rh2transit-just-transit.conf,production/application-sfbay/rh2transit-just-rh.conf,production/application-sfbay/rh2transit-plus10.conf,production/application-sfbay/rh2transit-off.conf -PbeamCommit=3cf29f1c6fd6ca2c5bed5c9995da10e8cd92de25 -Pbatch=false -PrunName=rideHailTransitExperiment + + +gradle deploy -PrunName=diffusion-base-fine-tuning-rhti2-wki-6 -PbeamCommit=5cf8dfd62b50aa148c41045bb69272d13396d2a1 + +gradle deploy -PrunName=diffusion-base-fine-tuning-rhti5-wki-6 -PbeamCommit=25f692c31e8340892703446a9c17391741d032b0 + +gradle deploy -PrunName=diffusion-base-fine-tuning-rhti10-wki-6 -PbeamCommit=7b9219e5b0e93d322bb1944bdab29ce9afab21d1 + +gradle deploy -PrunName=diffusion-base-fine-tuning-rhti15-wki-6 -PbeamCommit=541ebce2112f0b1647cb798d4a4dcece09aec927 + +gradle deploy -PrunName=diffusion-base-fine-tuning-rhti2-wki-7 -PbeamCommit=eff3a4510121e12e9873b0424f6d1a27c53bd41a + +gradle deploy -PrunName=diffusion-base-fine-tuning-rhti5-wki-7 -PbeamCommit=107e598f23c2838bda93c24b716f1ea8fb0f9aee + +gradle deploy -PrunName=diffusion-base-fine-tuning-rhti10-wki-7 -PbeamCommit=a3cb8776f43e1b6dcecacc498df2f05bb29ecd95 + +gradle deploy -PrunName=diffusion-base-fine-tuning-rhti15-wki-7 -PbeamCommit=bd74b4f288b262771e1dfdeedc76b76bbac9006b + +gradle deploy -PrunName=diffusion-base-fine-tuning-rhti2-wki-8 -PbeamCommit=121a272ddb29e001fcbf6ab0cc3eecf97ecfa534 + +gradle deploy -PrunName=diffusion-base-fine-tuning-rhti5-wki-8 -PbeamCommit=b32c572552a2714f1f05fd2e6651a296e8aec61e + +gradle deploy -PrunName=diffusion-base-fine-tuning-rhti10-wki-8 -PbeamCommit=c11d8d931de0154b4353435127bcb10e8a5ea3dc + +gradle deploy -PrunName=diffusion-base-fine-tuning-rhti15-wki-8 -PbeamCommit=423c5549d150efe7c2b72eff8c2dbe9e83a82de6 diff --git a/src/main/bash/listModes.sh b/src/main/bash/listModes.sh index 5f1a30f75b7..91205d45f5f 100644 --- a/src/main/bash/listModes.sh +++ b/src/main/bash/listModes.sh @@ -3,7 +3,6 @@ cd git/beam/production/application-sfbay/calibration/experiments/*/suggestions for d in */; do echo ${d::-1} - gunzip -c ${d::-1}/ITERS/it.0/0.events.xml.gz - grep ModeChoice - grep -Eo "mode=\"\w" sort uniq -c; + gunzip -c ${d::-1}/ITERS/it.0/0.events.xml.gz | grep ModeChoice + #grep -Eo "mode=\"\w" sort uniq -c; done diff --git a/src/main/bash/pop-attrib.sh b/src/main/bash/pop-attrib.sh new file mode 100644 index 00000000000..e5bf886e15d --- /dev/null +++ b/src/main/bash/pop-attrib.sh @@ -0,0 +1,16 @@ + +grep "object id" /Users/critter/Dropbox/ucb/vto/beam-all/beam/production/application-sfbay/samples/populationAttributes.xml | sed 's/.*id="//' | sed 's/">*//' > vott-id.csv +grep "valueOfTime" /Users/critter/Dropbox/ucb/vto/beam-all/beam/production/application-sfbay/samples/populationAttributes.xml | sed 's/.*Double">//' | sed 's/<\/attribute>*//' > vott-value.csv + +grep "object id" /Users/critter/Dropbox/ucb/vto/beam-all/beam/production/application-sfbay/samples/householdAttributes.xml | sed 's/.*id="//' | sed 's/">*//' > hh-id.csv +grep "homecoordx" /Users/critter/Dropbox/ucb/vto/beam-all/beam/production/application-sfbay/samples/householdAttributes.xml | sed 's/.*Double">//' | sed 's/<\/attribute>*//' > hh-x.csv +grep "homecoordy" /Users/critter/Dropbox/ucb/vto/beam-all/beam/production/application-sfbay/samples/householdAttributes.xml | sed 's/.*Double">//' | sed 's/<\/attribute>*//' > hh-y.csv +grep "housingtype" /Users/critter/Dropbox/ucb/vto/beam-all/beam/production/application-sfbay/samples/householdAttributes.xml | sed 's/.*Double">//' | sed 's/<\/attribute>*//' > hh-type.csv + +# following requires some editing +grep "//' | sed 's/.*household id="//' | sed 's/">.*//' > hh-members.csv + +# move them all back up + +mv vott*.csv /Users/critter/Dropbox/ucb/vto/beam-all/beam/production/application-sfbay/samples/ +mv hh*.csv /Users/critter/Dropbox/ucb/vto/beam-all/beam/production/application-sfbay/samples/ diff --git a/src/main/java/beam/agentsim/events/PathTraversalEvent.java b/src/main/java/beam/agentsim/events/PathTraversalEvent.java index 5c6dec2941f..105142659cc 100755 --- a/src/main/java/beam/agentsim/events/PathTraversalEvent.java +++ b/src/main/java/beam/agentsim/events/PathTraversalEvent.java @@ -53,7 +53,7 @@ public class PathTraversalEvent extends Event { private final double endY; public PathTraversalEvent(double time, Id vehicleId, BeamVehicleType vehicleType, Integer numPass, RoutingModel.BeamLeg beamLeg, double fuelConsumed, double endLegFuelLevel) { - this(time, vehicleId, vehicleType.vehicleCategory(), beamLeg.mode().value(), numPass, endLegFuelLevel, + this(time, vehicleId, vehicleType.vehicleTypeId(), beamLeg.mode().value(), numPass, endLegFuelLevel, (int)(vehicleType.seatingCapacity() + vehicleType.standingRoomCapacity()), fuelConsumed, beamLeg.travelPath().distanceInM(), beamLeg.travelPath().linkIds().mkString(","), beamLeg.startTime(), beamLeg.endTime(), diff --git a/src/main/java/beam/analysis/physsim/PhyssimCalcLinkSpeedDistributionStats.java b/src/main/java/beam/analysis/physsim/PhyssimCalcLinkSpeedDistributionStats.java new file mode 100644 index 00000000000..18f2860dfa4 --- /dev/null +++ b/src/main/java/beam/analysis/physsim/PhyssimCalcLinkSpeedDistributionStats.java @@ -0,0 +1,298 @@ +package beam.analysis.physsim; + +import beam.sim.config.BeamConfig; +import org.jfree.chart.ChartFactory; +import org.jfree.chart.ChartUtilities; +import org.jfree.chart.JFreeChart; +import org.jfree.chart.plot.PlotOrientation; +import org.jfree.data.category.CategoryDataset; +import org.jfree.data.category.DefaultCategoryDataset; +import org.jfree.data.general.DatasetUtilities; +import org.matsim.api.core.v01.network.Link; +import org.matsim.api.core.v01.network.Network; +import org.matsim.core.api.experimental.events.EventsManager; +import org.matsim.core.controler.OutputDirectoryHierarchy; +import org.matsim.core.router.util.TravelTime; +import org.matsim.core.trafficmonitoring.TravelTimeCalculator; +import org.matsim.core.utils.misc.Time; + +import java.awt.*; +import java.io.BufferedWriter; +import java.io.File; +import java.io.FileWriter; +import java.io.IOException; +import java.util.Comparator; +import java.util.HashMap; +import java.util.Map; +import java.util.stream.IntStream; +import java.util.stream.Stream; + +/** + * @author Bhavya Latha Bandaru. + * This class computes the distribution of free flow speed (in both m/s and %) over the network. + */ +public class PhyssimCalcLinkSpeedDistributionStats { + + private static int noOfBins = 24; + private BeamConfig beamConfig; + private Network network; + private OutputDirectoryHierarchy outputDirectoryHierarchy; + private String outputAsSpeedUnitFileName = "freeFlowSpeedDistribution"; + private String outputAsPercentageFileName = "freeFlowSpeedDistributionAsPercentage"; + + public PhyssimCalcLinkSpeedDistributionStats(Network network, OutputDirectoryHierarchy outputDirectoryHierarchy, BeamConfig beamConfig) { + this.network = network; + this.outputDirectoryHierarchy = outputDirectoryHierarchy; + this.beamConfig = beamConfig; + + // If not test mode pick up bin count from the beam configuration. + if (isNotTestMode()) { + Double endTime = Time.parseTime(beamConfig.matsim().modules().qsim().endTime()); + Double noOfTimeBins = endTime / this.beamConfig.beam().physsim().linkStatsBinSize(); + noOfTimeBins = Math.floor(noOfTimeBins); + noOfBins = noOfTimeBins.intValue() + 1; + } + } + + /** + * Iteration stop notification event listener + * @param iteration the count of the current iteration + */ + public void notifyIterationEnds(int iteration,TravelTimeCalculator travelTimeCalculator) { + //generate the graph input for the free flow speed distribution + Map processedSpeedDistributionData = generateInputDataForFreeFlowSpeedGraph(noOfBins,this.network); + //generate data matrix for the free flow speed distribution + double[][] speedDataMatrix = buildDataSetFromSpeedData(processedSpeedDistributionData); + //generate the graph input for the link efficiencies + Map processedSpeedDistributionAsPercentageData = generateInputDataForLinkEfficiencies(travelTimeCalculator); + //generate category data set for free flow speed distribution + CategoryDataset dataSetForSpeed = DatasetUtilities.createCategoryDataset("Free Speed", "", speedDataMatrix); + //generate the category data set for link efficiencies + CategoryDataset dataSetForSpeedAsPercentage = generateLinkEfficienciesDataSet(processedSpeedDistributionAsPercentageData); + if (this.outputDirectoryHierarchy != null) { + //If not running in test mode , write output to a csv file + if (isNotTestMode()) { + //write data outputs to CSV + this.writeCSV(speedDataMatrix,outputDirectoryHierarchy.getIterationFilename(iteration, outputAsSpeedUnitFileName+".csv"),"Free Speed Distribution(m/s)"); + this.writeCSV(processedSpeedDistributionAsPercentageData,outputDirectoryHierarchy.getIterationFilename(iteration, outputAsPercentageFileName+".csv"),"Free Speed Distribution(%)"); + } + //generate the required charts - frequency over speed (as m/s) + generateSpeedDistributionBarChart(dataSetForSpeed,iteration); + generateSpeedDistributionAsPercentageChart(dataSetForSpeedAsPercentage,iteration); + } + } + + private CategoryDataset generateLinkEfficienciesDataSet(Map generatedDataMap) { + final DefaultCategoryDataset dataSet = new DefaultCategoryDataset(); + try { + Map converterMap = new HashMap<>(); + generatedDataMap.forEach((k, v) -> { + int category = (int) Math.round(k) / 10; + category = ((category >= 0 && category != 10) ? (category + 1) : category) * 10; + Integer value = converterMap.getOrDefault(category, 0); + converterMap.put(category, value + v); + }); + IntStream.rangeClosed(1,10).forEach(i -> dataSet.addValue(converterMap.getOrDefault(i*10,0),"percentage",String.valueOf(i*10))); + } catch (Exception e) { + e.printStackTrace(); + } + return dataSet; + } + + /** + * Helper method that writes the final data to a CSV file + * @param dataMatrix the input data required to generate the charts + * @param outputFilePath path to the CSV file + * @param heading header string for the CSV file + */ + private void writeCSV(double[][] dataMatrix,String outputFilePath,String heading) { + try { + BufferedWriter bw = new BufferedWriter(new FileWriter(outputFilePath)); + String completeHeading = heading + ",x-coordinate,y-coordinate\n"; + bw.write(completeHeading); + double[] data = dataMatrix[0]; + IntStream.range(0,data.length) + .forEach( i -> { + try { + bw.write( i + "," + i + "," + data[i] + "\n"); + } catch (IOException e) { + e.printStackTrace(); + } + }); + bw.close(); + } catch (Exception e) { + e.printStackTrace(); + } + } + + /** + * Helper method that writes the final data to a CSV file + * @param dataMap the input data required to generate the charts + * @param outputFilePath path to the CSV file + * @param heading header string for the CSV file + */ + private void writeCSV(Map dataMap,String outputFilePath,String heading) { + try { + BufferedWriter bw = new BufferedWriter(new FileWriter(outputFilePath)); + String completeHeading = heading + ",x-coordinate,y-coordinate\n"; + bw.write(completeHeading); + dataMap.forEach((k,v) -> { + try { + bw.write( k + "," + (int)Math.round(k) + "," + v + "\n"); + } catch (IOException e) { + e.printStackTrace(); + } + }); + bw.close(); + } catch (Exception e) { + e.printStackTrace(); + } + } + + // A helper method to test if the application is running in test mode or not + private boolean isNotTestMode() { + return beamConfig != null; + } + + /** + * Generates input data used to generate free flow speed distribution chart + * @return input generated data as map ( speed in m/s -> frequency ) + */ + public Map generateInputDataForFreeFlowSpeedGraph(int binsCount,Network network) { + Map freeFlowSpeedFrequencies = new HashMap<>(); + Stream.iterate(0,x -> x) + .limit(binsCount) + .forEach(bin -> network.getLinks().values() + .stream() + .map(link -> link.getFreespeed(bin * 3600)) + .map(fs -> (int) Math.round(fs)) + .forEach(freeSpeed -> { + Integer frequencyCount = freeFlowSpeedFrequencies.getOrDefault(freeSpeed,0); + freeFlowSpeedFrequencies.put(freeSpeed,frequencyCount+1); + })); + return freeFlowSpeedFrequencies; + } + + public Stream getDistinctFreeSpeeds(int binsCount,Network network) { + return Stream.iterate(0, x -> x) + .limit(binsCount) + .flatMap(bin -> network.getLinks().values() + .stream() + .map(link -> link.getFreespeed(bin * 3600))) + .distinct(); + } + + /** + * Generates input data used to generate frequencies of link efficiencies + * @return input generated data as map ( speed in m/s -> frequency ) + */ + private Map generateInputDataForLinkEfficiencies(TravelTimeCalculator travelTimeCalculator) { + int binSize = 3600; + TravelTime travelTime = travelTimeCalculator.getLinkTravelTimes(); + Map frequencyOfEfficiencies = new HashMap<>(); + Map frequencyOfEfficiencies1 = new HashMap<>(); + //for each bin + for (int idx = 0; idx < noOfBins; idx++) { + //for each link + for (Link link : this.network.getLinks().values()) { + double freeSpeed = link.getFreespeed(idx * binSize); + double linkLength = link.getLength(); + double averageTime = travelTime.getLinkTravelTime(link, idx * binSize, null, null); + double averageSpeed = linkLength / averageTime; + //calculate the average speed of the link + double averageSpeedToFreeSpeedRatio = averageSpeed / freeSpeed; + Integer frequencyCount = frequencyOfEfficiencies.getOrDefault((int) Math.round(averageSpeedToFreeSpeedRatio*100),0); + Integer frequencyCount1 = frequencyOfEfficiencies1.getOrDefault(averageSpeedToFreeSpeedRatio*100,0); + frequencyOfEfficiencies.put((int) Math.round(averageSpeedToFreeSpeedRatio*100),frequencyCount+1); + frequencyOfEfficiencies1.put(averageSpeedToFreeSpeedRatio*100,frequencyCount1+1); + } + } + return frequencyOfEfficiencies1; + } + + /** + * Generate a 2d data matrix , used to generate category data set for stacked bar chart + * @param generatedDataMap input data generated as map + * @return 2d data matrix + */ + private double[][] buildDataSetFromSpeedData(Map generatedDataMap) { + Stream keys = generatedDataMap.keySet() + .stream(); + Integer max = keys.max(Comparator.comparing(Integer::valueOf)).orElse(0); + double[][] dataMatrix = new double[1][max+1]; + for (int i = 1; i <= max; i++) { + dataMatrix[0][i-1] = generatedDataMap.getOrDefault(i,0); + } + return dataMatrix; + } + + /** + * Generates a free flow speed (as m/s) distribution stacked bar chart + * @param dataSet the input data set for the chart + * @param iterationNumber The number of current iteration + */ + private void generateSpeedDistributionBarChart(CategoryDataset dataSet, int iterationNumber) { + // Settings legend and title for the plot + String plotTitle = "Free Flow Speed Distribution"; + String x_axis = "Free Speed (m/s)"; + String y_axis = "Frequency"; + int width = 1000; + int height = 600; + + // Setting orientation for the plot + PlotOrientation orientation = PlotOrientation.VERTICAL; + + // Create the chart + final JFreeChart chart = ChartFactory + .createStackedBarChart(plotTitle, x_axis, y_axis, dataSet, orientation, false, true, true); + chart.setBackgroundPaint(new Color(255, 255, 255)); + + //Save the chart as image + String graphImageFile = outputDirectoryHierarchy.getIterationFilename(iterationNumber, outputAsSpeedUnitFileName+".png"); + try { + ChartUtilities.saveChartAsPNG(new File(graphImageFile), chart, width, + height); + } catch (IOException e) { + e.printStackTrace(); + } + } + + /** + * Generates a free flow speed (as %) distribution line chart + * @param dataSet the input data set for the chart + * @param iterationNumber The number of current iteration + */ + private void generateSpeedDistributionAsPercentageChart(CategoryDataset dataSet, int iterationNumber) { + // Settings legend and title for the plot + String plotTitle = "Free Flow Speed Distribution (as percentage)"; + String x_axis = "Free Speed (%)"; + String y_axis = "Frequency"; + int width = 1000; + int height = 800; + // Setting orientation for the plot + PlotOrientation orientation = PlotOrientation.VERTICAL; + + // Create the chart + final JFreeChart chart = ChartFactory + .createStackedBarChart(plotTitle, x_axis, y_axis, dataSet, orientation, false, true, true); + chart.setBackgroundPaint(new Color(255, 255, 255)); + + //Save the chart as image + String graphImageFile = outputDirectoryHierarchy.getIterationFilename(iterationNumber, outputAsPercentageFileName+".png"); + try { + ChartUtilities.saveChartAsPNG(new File(graphImageFile), chart, width, + height); + } catch (IOException e) { + e.printStackTrace(); + } + } + + /** + * Iteration start notification event listener + * @param eventsManager The events manager class + */ + public void notifyIterationStarts(EventsManager eventsManager) { + } + + +} diff --git a/src/main/java/beam/analysis/physsim/PhyssimCalcLinkSpeedStats.java b/src/main/java/beam/analysis/physsim/PhyssimCalcLinkSpeedStats.java index f629cf5e8bc..d8e8fe20e60 100644 --- a/src/main/java/beam/analysis/physsim/PhyssimCalcLinkSpeedStats.java +++ b/src/main/java/beam/analysis/physsim/PhyssimCalcLinkSpeedStats.java @@ -195,4 +195,26 @@ private Color getRandomColor() { public void notifyIterationStarts(EventsManager eventsManager) { } + public double getAverageSpeedPercentageOfBin(int bin,TravelTimeCalculator travelTimeCalculator) { + try { + Map processedData = generateInputDataForGraph(travelTimeCalculator); + double[][] dataSet = buildDataSetFromProcessedData(processedData); + double[] averageSpeedPercentages = dataSet[0]; + return averageSpeedPercentages[bin]; + } catch (Exception e) { + e.printStackTrace(); + return 0; + } + } + + public double[] getAverageSpeedPercentagesOfAllBins(TravelTimeCalculator travelTimeCalculator) { + Map processedData = generateInputDataForGraph(travelTimeCalculator); + double[][] dataSet = buildDataSetFromProcessedData(processedData); + return dataSet[0]; + } + + public int getNumberOfBins() { + return noOfBins; + } + } diff --git a/src/main/java/beam/analysis/physsim/PhyssimCalcLinkStats.java b/src/main/java/beam/analysis/physsim/PhyssimCalcLinkStats.java index b20f4f34d34..c5964f4225d 100755 --- a/src/main/java/beam/analysis/physsim/PhyssimCalcLinkStats.java +++ b/src/main/java/beam/analysis/physsim/PhyssimCalcLinkStats.java @@ -76,7 +76,7 @@ public PhyssimCalcLinkStats(Network network, OutputDirectoryHierarchy controlerI noOfBins = _noOfTimeBins.intValue() + 1; } - linkStats = new BeamCalcLinkStats(network); + linkStats = new BeamCalcLinkStats(network,beamConfig.beam().physsim().quick_fix_minCarSpeedInMetersPerSecond()); } @@ -96,7 +96,7 @@ public void notifyIterationEnds(int iteration, TravelTimeCalculator travelTimeCa } private boolean isNotTestMode() { - return beamConfig != null; + return controllerIO != null; } diff --git a/src/main/java/beam/analysis/via/EventWriterXML_viaCompatible.java b/src/main/java/beam/analysis/via/EventWriterXML_viaCompatible.java index 8ac0ed8359f..018b8d89644 100755 --- a/src/main/java/beam/analysis/via/EventWriterXML_viaCompatible.java +++ b/src/main/java/beam/analysis/via/EventWriterXML_viaCompatible.java @@ -23,13 +23,15 @@ public class EventWriterXML_viaCompatible implements EventWriter, BasicEventHand private static final String BUS = "SF"; private static final String CAR = "car"; private final BufferedWriter out; + private boolean eventsForFullVersionOfVia; HashMap> filterPeopleForViaDemo = new HashMap<>(); HashMap maxPeopleForViaDemo = new HashMap<>(); private String outFileName; - public EventWriterXML_viaCompatible(final String outFileName) { + public EventWriterXML_viaCompatible(final String outFileName, boolean eventsForFullVersionOfVia) { this.outFileName = outFileName; this.out = IOUtils.getBufferedWriter(outFileName); + this.eventsForFullVersionOfVia = eventsForFullVersionOfVia; filterPeopleForViaDemo.put(CAR, new HashSet<>()); filterPeopleForViaDemo.put(BUS, new HashSet<>()); @@ -71,6 +73,11 @@ public void reset(final int iter) { } private boolean addPersonToEventsFile(String person) { + + if (eventsForFullVersionOfVia){ + return true; + } + String personLabel = null; if (person == null) { diff --git a/src/main/java/beam/physsim/jdeqsim/AgentSimToPhysSimPlanConverter.java b/src/main/java/beam/physsim/jdeqsim/AgentSimToPhysSimPlanConverter.java index 709de065236..f019ce9654e 100755 --- a/src/main/java/beam/physsim/jdeqsim/AgentSimToPhysSimPlanConverter.java +++ b/src/main/java/beam/physsim/jdeqsim/AgentSimToPhysSimPlanConverter.java @@ -2,6 +2,7 @@ import akka.actor.ActorRef; import beam.agentsim.events.PathTraversalEvent; +import beam.analysis.physsim.PhyssimCalcLinkSpeedDistributionStats; import beam.analysis.physsim.PhyssimCalcLinkSpeedStats; import beam.analysis.physsim.PhyssimCalcLinkStats; import beam.analysis.via.EventWriterXML_viaCompatible; @@ -43,6 +44,10 @@ import java.io.File; import java.util.*; import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; + /** @@ -55,6 +60,7 @@ public class AgentSimToPhysSimPlanConverter implements BasicEventHandler, Metric public static final String DUMMY_ACTIVITY = "DummyActivity"; private static PhyssimCalcLinkStats linkStatsGraph; private static PhyssimCalcLinkSpeedStats linkSpeedStatsGraph; + private static PhyssimCalcLinkSpeedDistributionStats linkSpeedDistributionStatsGraph; private final ActorRef router; private final OutputDirectoryHierarchy controlerIO; private Logger log = LoggerFactory.getLogger(AgentSimToPhysSimPlanConverter.class); @@ -69,6 +75,8 @@ public class AgentSimToPhysSimPlanConverter implements BasicEventHandler, Metric private boolean agentSimPhysSimInterfaceDebuggerEnabled; + private List linkStatsFutures= new ArrayList<>(); + public AgentSimToPhysSimPlanConverter(EventsManager eventsManager, TransportNetwork transportNetwork, OutputDirectoryHierarchy controlerIO, @@ -94,6 +102,7 @@ public AgentSimToPhysSimPlanConverter(EventsManager eventsManager, linkStatsGraph = new PhyssimCalcLinkStats(agentSimScenario.getNetwork(), controlerIO, beamConfig); linkSpeedStatsGraph = new PhyssimCalcLinkSpeedStats(agentSimScenario.getNetwork(),controlerIO,beamConfig); + linkSpeedDistributionStatsGraph = new PhyssimCalcLinkSpeedDistributionStats(agentSimScenario.getNetwork(),controlerIO,beamConfig); } private void preparePhysSimForNewIteration() { @@ -122,7 +131,7 @@ public void setupActorsAndRunPhysSim(int iterationNumber) { EventWriterXML_viaCompatible eventsWriterXML = null; if (writePhysSimEvents(iterationNumber)) { - eventsWriterXML = new EventWriterXML_viaCompatible(controlerIO.getIterationFilename(iterationNumber, "physSimEvents.xml.gz")); + eventsWriterXML = new EventWriterXML_viaCompatible(controlerIO.getIterationFilename(iterationNumber, "physSimEvents.xml.gz"), beamConfig.beam().physsim().eventsForFullVersionOfVia()); jdeqsimEvents.addHandler(eventsWriterXML); } @@ -134,6 +143,7 @@ public void setupActorsAndRunPhysSim(int iterationNumber) { linkStatsGraph.notifyIterationStarts(jdeqsimEvents); linkSpeedStatsGraph.notifyIterationStarts(jdeqsimEvents); + linkSpeedDistributionStatsGraph.notifyIterationStarts(jdeqsimEvents); log.info("JDEQSim Start"); startSegment("jdeqsim-execution", "jdeqsim"); @@ -168,14 +178,14 @@ public void setupActorsAndRunPhysSim(int iterationNumber) { } - CompletableFuture.runAsync(() -> { + linkStatsFutures.add(CompletableFuture.runAsync(() -> { linkStatsGraph.notifyIterationEnds(iterationNumber, travelTimeCalculator); linkStatsGraph.clean(); - }); + })); - CompletableFuture.runAsync(() -> { - linkSpeedStatsGraph.notifyIterationEnds(iterationNumber, travelTimeCalculator); - }); + CompletableFuture.runAsync(() -> + linkSpeedStatsGraph.notifyIterationEnds(iterationNumber, travelTimeCalculator) + ); if (writePhysSimEvents(iterationNumber)) { eventsWriterXML.closeFile(); @@ -186,6 +196,18 @@ public void setupActorsAndRunPhysSim(int iterationNumber) { jdeqSimScenario.setNetwork(null); jdeqSimScenario.setPopulation(null); + if (iterationNumber == beamConfig.matsim().modules().controler().lastIteration()) { + try { + CompletableFuture allOfLinStatFutures = CompletableFuture.allOf(linkStatsFutures.toArray(new CompletableFuture[0])); + log.info("Waiting started on link stats file dump."); + allOfLinStatFutures.get(20, TimeUnit.MINUTES); + log.info("Link stats file dump completed."); + + } catch (InterruptedException | ExecutionException | TimeoutException e) { + log.error("Error while generating link stats.",e); + } + } + router.tell(new BeamRouter.UpdateTravelTime(travelTimeCalculator.getLinkTravelTimes()), ActorRef.noSender()); } diff --git a/src/main/java/beam/utils/BeamCalcLinkStats.java b/src/main/java/beam/utils/BeamCalcLinkStats.java index 46d28768944..154a2f8432f 100755 --- a/src/main/java/beam/utils/BeamCalcLinkStats.java +++ b/src/main/java/beam/utils/BeamCalcLinkStats.java @@ -34,7 +34,7 @@ import java.io.BufferedWriter; import java.io.IOException; import java.util.Map; -import java.util.TreeMap; +import java.util.concurrent.ConcurrentHashMap; public class BeamCalcLinkStats { @@ -48,21 +48,15 @@ public class BeamCalcLinkStats { private final int nofHours; private final Network network; private int count = 0; + private double minCarSpeedInMetersPerSecond; @Inject - public BeamCalcLinkStats(final Network network) { + public BeamCalcLinkStats(final Network network, double minCarSpeedInMetersPerSecond) { this.network = network; - this.linkData = new TreeMap<>(); + this.linkData = new ConcurrentHashMap<>(); this.nofHours = 24; reset(); - } - - /** - * @param network - * @param vol_scale_factor scaling factor when reading in values from a file - */ - public BeamCalcLinkStats(final Network network, double vol_scale_factor) { - this(network); + this.minCarSpeedInMetersPerSecond = minCarSpeedInMetersPerSecond; } public void addData(final VolumesAnalyzer analyzer, final TravelTime ttimes) { @@ -216,14 +210,13 @@ public void writeFile(final String filename) { } else if (j == MAX && i < this.nofHours) { writeCommaAndStr(out, Double.toString(data.ttimes[MAX][i])); } + out.write("\n"); } - } - } - + out.flush(); } catch (IOException e) { e.printStackTrace(); } finally { @@ -242,14 +235,12 @@ private void writeCommaAndStr(BufferedWriter out, String str) throws IOException } private static class LinkData { - public final double[][] volumes; - public final double[][] ttimes; + final double[][] volumes; + final double[][] ttimes; - public LinkData(final double[][] linksVolumes, final double[][] linksTTimes) { + LinkData(final double[][] linksVolumes, final double[][] linksTTimes) { this.volumes = linksVolumes.clone(); this.ttimes = linksTTimes.clone(); } } - - } diff --git a/src/main/java/beam/utils/UnzipUtility.java b/src/main/java/beam/utils/UnzipUtility.java index e1c3f173461..0411b07d3bd 100755 --- a/src/main/java/beam/utils/UnzipUtility.java +++ b/src/main/java/beam/utils/UnzipUtility.java @@ -2,6 +2,7 @@ import java.io.*; import java.nio.file.Paths; +import java.util.zip.GZIPInputStream; import java.util.zip.ZipEntry; import java.util.zip.ZipInputStream; @@ -54,6 +55,36 @@ public static void unzip(String zipFilePath, String destDirectory, boolean delet } } + // TODO: double check, if this is redundant + /** + * Extracts a zip file specified by the zipFilePath to a directory specified by + * destDirectory (will be created if does not exists) + * + * @param compressedFile Source directory for zip file + * @param decompressedFile Target directory for unzipping. + * @throws IOException Error on failure. + */ + public static void unGunzipFile(String compressedFile, String decompressedFile, boolean delete) throws IOException { + FileInputStream fileIn = new FileInputStream(compressedFile); + GZIPInputStream gZIPInputStream = new GZIPInputStream(fileIn); + FileOutputStream fileOutputStream = new FileOutputStream(decompressedFile); + + byte[] buffer = new byte[BUFFER_SIZE]; + + int bytes_read; + + while ((bytes_read = gZIPInputStream.read(buffer)) > 0) { + fileOutputStream.write(buffer, 0, bytes_read); + } + + gZIPInputStream.close(); + fileOutputStream.close(); + + if (delete) { + delete(Paths.get(compressedFile)); + } + } + /** * Extracts a zip entry (file entry) * diff --git a/src/main/resources/beam-template.conf b/src/main/resources/beam-template.conf index fcc1efa38fc..d5192fb55e1 100755 --- a/src/main/resources/beam-template.conf +++ b/src/main/resources/beam-template.conf @@ -100,7 +100,8 @@ beam.physsim.linkStatsBinSize = "int | 3600" beam.physsim.ptSampleSize = "double | 1.0" beam.physsim.jdeqsim.agentSimPhysSimInterfaceDebugger.enabled = false beam.physsim.skipPhysSim = false - +beam.physsim.eventsForFullVersionOfVia = true +beam.physsim.quick_fix_minCarSpeedInMetersPerSecond = "double | 0.5" ################################################################## # Warm Mode ################################################################## diff --git a/src/main/scala/beam/agentsim/agents/modalbehaviors/ChoosesMode.scala b/src/main/scala/beam/agentsim/agents/modalbehaviors/ChoosesMode.scala index 5d58030d0e5..77ad15ae458 100755 --- a/src/main/scala/beam/agentsim/agents/modalbehaviors/ChoosesMode.scala +++ b/src/main/scala/beam/agentsim/agents/modalbehaviors/ChoosesMode.scala @@ -63,12 +63,15 @@ trait ChoosesMode { val currentPersonLocation = choosesModeData.currentLocation.getOrElse( SpaceTime(currentActivity(choosesModeData.personData).getCoord, _currentTick.get) ) - choosesModeData.personData.currentTourMode match { - case Some(mode) => - log.debug("{}", mode) - val currAct = currentActivity(choosesModeData.personData) - val i = 0 + val availableModes: Seq[BeamMode] = availableModesForPerson( + beamServices.matsimServices.getScenario.getPopulation.getPersons.get(id) + ) + // Make sure the current mode is allowable + val correctedCurrentTourMode = choosesModeData.personData.currentTourMode match { + case Some(mode) if availableModes.contains(mode) => + Some(mode) case _ => + None } val bodyStreetVehicle = StreetVehicle( @@ -80,12 +83,8 @@ trait ChoosesMode { val nextAct = nextActivity(choosesModeData.personData).right.get val departTime = DiscreteTime(_currentTick.get.toInt) - val availableModes: Seq[BeamMode] = availableModesForPerson( - beamServices.matsimServices.getScenario.getPopulation.getPersons.get(id) - ) - val availablePersonalStreetVehicles = - choosesModeData.personData.currentTourMode match { + correctedCurrentTourMode match { case None | Some(CAR | BIKE) => // In these cases, a personal vehicle will be involved streetVehicles.filter(_.asDriver) @@ -131,7 +130,7 @@ trait ChoosesMode { def makeRideHailTransitRoutingRequest(bodyStreetVehicle: StreetVehicle): Option[Int] = { //TODO make ride hail wait buffer config param - val startWithWaitBuffer = 600 + departTime.atTime + val startWithWaitBuffer = 900 + departTime.atTime val currentSpaceTime = SpaceTime(currentPersonLocation.loc, startWithWaitBuffer) val theRequest = RoutingRequest( @@ -160,24 +159,24 @@ trait ChoosesMode { } } - // Cache condition variables here to restrict modes to only those available - val hasRideHail = availableModes.contains(RIDE_HAIL) var responsePlaceholders = ChoosesModeResponsePlaceholders() var requestId: Option[Int] = None // Form and send requests - choosesModeData.personData.currentTourMode match { + correctedCurrentTourMode match { case None => if (hasRideHail) { responsePlaceholders = makeResponsePlaceholders( withRouting = true, withRideHail = true, - withRideHailTransit = true + withRideHailTransit = !choosesModeData.isWithinTripReplanning ) makeRideHailRequest() - requestId = makeRideHailTransitRoutingRequest(bodyStreetVehicle) + if (!choosesModeData.isWithinTripReplanning) { + requestId = makeRideHailTransitRoutingRequest(bodyStreetVehicle) + } } else { responsePlaceholders = makeResponsePlaceholders(withRouting = true) requestId = None @@ -270,6 +269,7 @@ trait ChoosesMode { logDebug(m.toString) } val newPersonData = choosesModeData.copy( + personData = choosesModeData.personData.copy(currentTourMode = correctedCurrentTourMode), availablePersonalStreetVehicles = availablePersonalStreetVehicles, routingResponse = responsePlaceholders.routingResponse, rideHail2TransitRoutingResponse = responsePlaceholders.rideHail2TransitRoutingResponse, @@ -438,7 +438,7 @@ trait ChoosesMode { ) ) ++ driveTransitTrip.legs.tail val fullTrip = if (rideHail2TransitEgressResult.error.isEmpty) { - accessAndTransit.dropRight(1) ++ rideHail2TransitEgressResult.travelProposal.head.responseRideHail2Dest.itineraries.head.legs.tail + accessAndTransit.dropRight(2) ++ rideHail2TransitEgressResult.travelProposal.head.responseRideHail2Dest.itineraries.head.legs.tail } else { accessAndTransit } @@ -488,7 +488,12 @@ trait ChoosesMode { } val combinedItinerariesForChoice = rideHailItinerary ++ routingResponse.itineraries ++ rideHail2TransitIinerary.toVector // val test = createRideHail2TransitItin(rideHail2TransitAccessResult, rideHail2TransitEgressResult, routingResponse) - val filteredItinerariesForChoice = personData.currentTourMode match { + + val availableModes: Seq[BeamMode] = availableModesForPerson( + beamServices.matsimServices.getScenario.getPopulation.getPersons.get(id) + ) + + val filteredItinerariesForChoice = (choosesModeData.personData.currentTourMode match { case Some(DRIVE_TRANSIT) => val LastTripIndex = currentTour(choosesModeData.personData).trips.size - 1 ( @@ -510,7 +515,7 @@ trait ChoosesMode { combinedItinerariesForChoice.filter(_.tripClassifier == mode) case _ => combinedItinerariesForChoice - } + }).filter(itin => availableModes.contains(itin.tripClassifier)) if (filteredItinerariesForChoice.size == 1 && filteredItinerariesForChoice.head.tripClassifier == WALK) { val i = 0 diff --git a/src/main/scala/beam/agentsim/agents/ridehail/RideHailManager.scala b/src/main/scala/beam/agentsim/agents/ridehail/RideHailManager.scala index 8e8349ce547..6b59acea908 100755 --- a/src/main/scala/beam/agentsim/agents/ridehail/RideHailManager.scala +++ b/src/main/scala/beam/agentsim/agents/ridehail/RideHailManager.scala @@ -212,7 +212,7 @@ class RideHailManager( val future = rideHailIterationHistoryActor.ask(GetCurrentIterationRideHailStats) Await - .result(future, timeout.duration) + .result(future, Timeout(60, TimeUnit.SECONDS).duration) .asInstanceOf[Option[TNCIterationStats]] } private val rideHailAllocationManagerTimeoutInSeconds = diff --git a/src/main/scala/beam/calibration/BeamSigoptTuner.scala b/src/main/scala/beam/calibration/BeamSigoptTuner.scala index 730cc1e0784..83b8a75cb2e 100755 --- a/src/main/scala/beam/calibration/BeamSigoptTuner.scala +++ b/src/main/scala/beam/calibration/BeamSigoptTuner.scala @@ -159,13 +159,15 @@ object BeamSigoptTuner { val parameter = new Parameter.Builder().name(paramName) + //println(maxValue) + // Build bounds maxValue match { - case _: Double => + case x if x.isInstanceOf[Double] => parameter .bounds(getBounds(minValue.asInstanceOf[Double], maxValue.asInstanceOf[Double])) .`type`("double") - case _: Int => + case x if x.isInstanceOf[Int] => parameter .`type`("int") .bounds(getBounds(minValue.asInstanceOf[Int], maxValue.asInstanceOf[Int])) diff --git a/src/main/scala/beam/calibration/ExperimentRunner.scala b/src/main/scala/beam/calibration/ExperimentRunner.scala index ef82274899f..c23806dcd1f 100755 --- a/src/main/scala/beam/calibration/ExperimentRunner.scala +++ b/src/main/scala/beam/calibration/ExperimentRunner.scala @@ -8,7 +8,12 @@ import com.sigopt.model.{Observation, Suggestion} import com.typesafe.config.{Config, ConfigValueFactory} import beam.analysis.plots.GraphsStatsAgentSimEventsListener import beam.calibration.api.{FileBasedObjectiveFunction, ObjectiveFunction} -import beam.calibration.impl.example.{CountsObjectiveFunction, ErrorComparisonType, ModeChoiceObjectiveFunction} +import beam.calibration.impl.example.{ + CountsObjectiveFunction, + ErrorComparisonType, + ModeChoiceObjectiveFunction, + RideHailObjectiveFunction +} import beam.sim.BeamHelper import beam.utils.reflection.ReflectionUtils @@ -97,6 +102,16 @@ case class ExperimentRunner(implicit experimentData: SigoptExperimentData) exten outpath.toAbsolutePath.toString, ErrorComparisonType.AbsoluteErrorWithPreferenceForModeDiversity ) + } else if (objectiveFunctionClassName.equals( + "ModeChoiceObjectiveFunction_AbsoluteErrorWithMinLevelRepresentationOfMode" + )) { + val benchmarkData = Paths.get(experimentData.benchmarkFileLoc).toAbsolutePath + val outpath = Paths.get(GraphsStatsAgentSimEventsListener.CONTROLLER_IO.getOutputFilename("modeChoice.csv")) + new ModeChoiceObjectiveFunction(benchmarkData.toAbsolutePath.toString) + .evaluateFromRun( + outpath.toAbsolutePath.toString, + ErrorComparisonType.AbsoluteErrorWithMinLevelRepresentationOfMode + ) } else if (objectiveFunctionClassName.equals("ModeChoiceAndCountsObjectiveFunction")) { var outpath = Paths.get( GraphsStatsAgentSimEventsListener.CONTROLLER_IO @@ -117,6 +132,11 @@ case class ExperimentRunner(implicit experimentData: SigoptExperimentData) exten val countsWeight = 1 - modeWeight -(countsWeight * Math.abs(countsObjVal) + modeWeight * Math.abs(modesObjVal)) + } else if (objectiveFunctionClassName.equals( + "RideHail_maximizeReservationCount" + )) { + val outpath = Paths.get(GraphsStatsAgentSimEventsListener.CONTROLLER_IO.getOutputFilename("ridehailStats.csv")) + RideHailObjectiveFunction.evaluateFromRun(outpath.toAbsolutePath.toString) } else { logger.error("objectiveFunctionClassName not set") Double.NegativeInfinity diff --git a/src/main/scala/beam/calibration/impl/example/ModeChoiceObjectiveFunction.scala b/src/main/scala/beam/calibration/impl/example/ModeChoiceObjectiveFunction.scala index 13d806aa0a9..188be6bd715 100755 --- a/src/main/scala/beam/calibration/impl/example/ModeChoiceObjectiveFunction.scala +++ b/src/main/scala/beam/calibration/impl/example/ModeChoiceObjectiveFunction.scala @@ -30,9 +30,49 @@ class ModeChoiceObjectiveFunction(benchmarkDataFileLoc: String) { compareStatsAbsolutError(benchmarkData, getStatsFromFile(runDataFileLoc)) } else if (comparisonType == ErrorComparisonType.AbsoluteErrorWithPreferenceForModeDiversity) { compareStatsAbsolutError(benchmarkData, getStatsFromFile(runDataFileLoc)) + getStatsFromFile(runDataFileLoc).size * 0.1 + } else if (comparisonType == ErrorComparisonType.AbsoluteErrorWithMinLevelRepresentationOfMode) { + val runModeStats = getStatsFromFile(runDataFileLoc) + var objective = compareStatsAbsolutError(benchmarkData, runModeStats) + getStatsFromFile(runDataFileLoc).size * 0.1 + + if (minLevelRepresentationOfMode(runModeStats, benchmarkData, 0.8, "car")) { + objective = objective + 0.1 + } + + if (minLevelRepresentationOfMode(runModeStats, benchmarkData, 0.3, "drive_transit")) { + objective = objective + 0.1 + } + + if (minLevelRepresentationOfMode(runModeStats, benchmarkData, 0.3, "ride_hail")) { + objective = objective + 0.1 + } + + if (minLevelRepresentationOfMode(runModeStats, benchmarkData, 0.3, "ride_hail_transit")) { + objective = objective + 0.1 + } + + if (minLevelRepresentationOfMode(runModeStats, benchmarkData, 0.3, "walk")) { + objective = objective + 0.1 + } + + if (minLevelRepresentationOfMode(runModeStats, benchmarkData, 0.3, "walk_transit")) { + objective = objective + 0.1 + } + + objective } else { compareStatsRMSPE(benchmarkData, getStatsFromFile(runDataFileLoc)) } + + } + + def minLevelRepresentationOfMode( + runModeStats: Map[String, Double], + benchmarkData: Map[String, Double], + minLevelRepresentationOfMode: Double, + mode: String + ): Boolean = { + runModeStats.contains(mode) && (runModeStats(mode) + - benchmarkData(mode) * minLevelRepresentationOfMode) > 0 } def compareStatsAbsolutError( @@ -127,5 +167,7 @@ object ModeChoiceObjectiveFunction { } object ErrorComparisonType extends Enumeration { - val RMSPE, AbsoluteError, AbsoluteErrorWithPreferenceForModeDiversity = Value + + val RMSPE, AbsoluteError, AbsoluteErrorWithPreferenceForModeDiversity, AbsoluteErrorWithMinLevelRepresentationOfMode = + Value } diff --git a/src/main/scala/beam/calibration/impl/example/RideHailObjectiveFunction.scala b/src/main/scala/beam/calibration/impl/example/RideHailObjectiveFunction.scala new file mode 100644 index 00000000000..a232865bc3e --- /dev/null +++ b/src/main/scala/beam/calibration/impl/example/RideHailObjectiveFunction.scala @@ -0,0 +1,24 @@ +package beam.calibration.impl.example + +import scala.io.Source + +import beam.calibration.api.ObjectiveFunction +import beam.utils.FileUtils.using + +object RideHailObjectiveFunction { + + def evaluateFromRun(runDataPath: String): Double = { + val rideHailStats = getStatsFromFile(runDataPath) + val reservationCount = rideHailStats("reservationCount") + reservationCount + } + + def getStatsFromFile(fileLoc: String): Map[String, Double] = { + val lines = Source.fromFile(fileLoc).getLines().toArray + val header = lines.head.split(",").tail + val lastIter = lines.reverse.head.split(",").tail.map(_.toDouble) + val total = lastIter.sum + val pcts = lastIter.map(x => x / total) + header.zip(pcts).toMap + } +} diff --git a/src/main/scala/beam/calibration/utils/CreateExperiment.scala b/src/main/scala/beam/calibration/utils/CreateExperiment.scala index 0376a61e7c1..e5432f05cac 100644 --- a/src/main/scala/beam/calibration/utils/CreateExperiment.scala +++ b/src/main/scala/beam/calibration/utils/CreateExperiment.scala @@ -12,7 +12,7 @@ object CreateExperiment extends LazyLogging { def main(args: Array[String]): Unit = { val experimentLoc: String = - "production/application-sfbay/calibration/experiment_modes_and_counts_calibration_intercept_and_network.yml" + "production/application-sfbay/calibration/experiment_modes_calibration.yml" val benchmarkLoc: String = "production/application-sfbay/calibration/benchmark.csv" SigoptExperimentData(experimentLoc, benchmarkLoc, NEW_EXPERIMENT_FLAG, development = false) } diff --git a/src/main/scala/beam/router/r5/R5RoutingWorker.scala b/src/main/scala/beam/router/r5/R5RoutingWorker.scala index 42cbcbb3e03..acc4f6d5646 100755 --- a/src/main/scala/beam/router/r5/R5RoutingWorker.scala +++ b/src/main/scala/beam/router/r5/R5RoutingWorker.scala @@ -10,6 +10,8 @@ import akka.actor._ import akka.pattern._ import beam.agentsim.agents.household.HouseholdActor import beam.agentsim.agents.modalbehaviors.ModeChoiceCalculator +import beam.agentsim.agents.vehicles.{BeamVehicle, BeamVehicleType, FuelType} +import beam.agentsim.agents.vehicles.EnergyEconomyAttributes.Powertrain import beam.agentsim.agents.vehicles.VehicleProtocol.StreetVehicle import beam.agentsim.agents.vehicles.{BeamVehicle, BeamVehicleType, FuelType} import beam.agentsim.events.SpaceTime @@ -254,14 +256,7 @@ class R5RoutingWorker(workerParams: WorkerParameters) extends Actor with ActorLo val travelTime = (time: Int, linkId: Int) => maybeTravelTime match { case Some(matsimTravelTime) => - matsimTravelTime - .getLinkTravelTime( - network.getLinks.get(Id.createLinkId(linkId)), - time.toDouble, - null, - null - ) - .toInt + getTravelTime(time, linkId, matsimTravelTime).toInt case None => val edge = transportNetwork.streetLayer.edgeStore.getCursor(linkId) (edge.getLengthM / edge.calculateSpeed( @@ -385,7 +380,7 @@ class R5RoutingWorker(workerParams: WorkerParameters) extends Actor with ActorLo profileRequest.accessModes = util.EnumSet.of(request.accessMode) profileRequest.egressModes = util.EnumSet.of(request.egressMode) } -// log.debug(profileRequest.toString) + // log.debug(profileRequest.toString) val result = try { getPlan(profileRequest) } catch { @@ -394,12 +389,12 @@ class R5RoutingWorker(workerParams: WorkerParameters) extends Actor with ActorLo case _: ArrayIndexOutOfBoundsException => new ProfileResponse } -// log.debug(s"# options found = ${result.options.size()}") + // log.debug(s"# options found = ${result.options.size()}") result } def calcRoute(routingRequest: RoutingRequest): RoutingResponse = { -// log.debug(routingRequest.toString) + // log.debug(routingRequest.toString) // For each street vehicle (including body, if available): Route from origin to street vehicle, from street vehicle to destination. val isRouteForPerson = routingRequest.streetVehicles.exists(_.mode == WALK) @@ -802,10 +797,10 @@ class R5RoutingWorker(workerParams: WorkerParameters) extends Actor with ActorLo routingRequest.streetVehicles.flatMap(vehicle => tripsForVehicle(vehicle)) if (!embodiedTrips.exists(_.tripClassifier == WALK)) { -// log.debug("No walk route found. {}", routingRequest) + // log.debug("No walk route found. {}", routingRequest) val maybeBody = routingRequest.streetVehicles.find(_.mode == WALK) if (maybeBody.isDefined) { -// log.debug("Adding dummy walk route with maximum street time.") + // log.debug("Adding dummy walk route with maximum street time.") val dummyTrip = R5RoutingWorker.createBushwackingTrip( beamServices.geo.utm2Wgs(new Coord(routingRequest.origin.getX, routingRequest.origin.getY)), beamServices.geo.utm2Wgs(new Coord(routingRequest.destination.getX, routingRequest.destination.getY)), @@ -819,7 +814,7 @@ class R5RoutingWorker(workerParams: WorkerParameters) extends Actor with ActorLo Some(routingRequest.requestId) ) } else { -// log.debug("Not adding a dummy walk route since agent has no body.") + // log.debug("Not adding a dummy walk route since agent has no body.") RoutingResponse( embodiedTrips, routingRequest.staticRequestId, @@ -1026,29 +1021,17 @@ class R5RoutingWorker(workerParams: WorkerParameters) extends Actor with ActorLo // MATSim network. (edge.getLengthM / edge.calculateSpeed(req, streetMode)).toFloat } else { - travelTime - .getLinkTravelTime( - network.getLinks.get(Id.createLinkId(edge.getEdgeIndex)), - startTime + durationSeconds, - null, - null - ) - .asInstanceOf[Float] + getTravelTime(startTime + durationSeconds, edge.getEdgeIndex, travelTime).toFloat } } case None => new EdgeStore.DefaultTravelTimeCalculator } + private def travelTimeByLinkCalculator(time: Int, linkId: Int, mode: StreetMode): Int = { maybeTravelTime match { case Some(matsimTravelTime) if mode == StreetMode.CAR => - matsimTravelTime - .getLinkTravelTime( - network.getLinks.get(Id.createLinkId(linkId)), - time.toDouble, - null, - null - ) - .toInt + getTravelTime(time, linkId, matsimTravelTime).toInt + case _ => val edge = transportNetwork.streetLayer.edgeStore.getCursor(linkId) // (new EdgeStore.DefaultTravelTimeCalculator).getTravelTimeMilliseconds(edge,) @@ -1057,6 +1040,26 @@ class R5RoutingWorker(workerParams: WorkerParameters) extends Actor with ActorLo } } + private def getTravelTime(time: Int, linkId: Int, matsimTravelTime: TravelTime): Double = { + var travelTime = matsimTravelTime + .getLinkTravelTime( + network.getLinks.get(Id.createLinkId(linkId)), + time.toDouble, + null, + null + ) + + val travelSpeed = network.getLinks.get(Id.createLinkId(linkId)).getLength / travelTime + if (travelSpeed < beamServices.beamConfig.beam.physsim.quick_fix_minCarSpeedInMetersPerSecond) { + network.getLinks + .get(Id.createLinkId(linkId)) + .getLength / beamServices.beamConfig.beam.physsim.quick_fix_minCarSpeedInMetersPerSecond + } else { + travelTime + } + + } + private val turnCostCalculator: TurnCostCalculator = new TurnCostCalculator(transportNetwork.streetLayer, true) { override def computeTurnCost(fromEdge: Int, toEdge: Int, streetMode: StreetMode): Int = 0 @@ -1095,13 +1098,13 @@ class R5RoutingWorker(workerParams: WorkerParameters) extends Actor with ActorLo new StreetSegment(streetPath, mode, transportNetwork.streetLayer) option.addDirect(streetSegment, request.getFromTimeDateZD) } else { -// log.debug("Direct mode {} last state wasn't found", mode) + // log.debug("Direct mode {} last state wasn't found", mode) } } else { -// log.debug("Direct mode {} destination wasn't found!", mode) + // log.debug("Direct mode {} destination wasn't found!", mode) } } else { -// log.debug("Direct mode {} origin wasn't found!", mode) + // log.debug("Direct mode {} origin wasn't found!", mode) } } option.summary = option.generateSummary @@ -1152,7 +1155,7 @@ class R5RoutingWorker(workerParams: WorkerParameters) extends Actor with ActorLo foo(o1, o2) }) -// log.debug("Usefull paths:{}", usefullpathList.size) + // log.debug("Usefull paths:{}", usefullpathList.size) for (path <- usefullpathList.asScala) { profileResponse.addTransitPath( @@ -1168,8 +1171,8 @@ class R5RoutingWorker(workerParams: WorkerParameters) extends Actor with ActorLo } // latency possible candidate } profileResponse.recomputeStats(request) -// log.debug("Returned {} options", profileResponse.getOptions.size) -// log.debug("Took {} ms", System.currentTimeMillis - startRouting) + // log.debug("Returned {} options", profileResponse.getOptions.size) + // log.debug("Took {} ms", System.currentTimeMillis - startRouting) profileResponse } diff --git a/src/main/scala/beam/sim/BeamHelper.scala b/src/main/scala/beam/sim/BeamHelper.scala index 0ebd7b5e66b..196b3d9a68b 100755 --- a/src/main/scala/beam/sim/BeamHelper.scala +++ b/src/main/scala/beam/sim/BeamHelper.scala @@ -362,6 +362,9 @@ trait BeamHelper extends LazyLogging { val networkCoordinator = new NetworkCoordinator(beamConfig) networkCoordinator.loadNetwork() + val beamWarmStart = BeamWarmStart(beamConfig) + beamWarmStart.warmStartPopulationIfNeeded(matsimConfig) + val scenario = ScenarioUtils.loadScenario(matsimConfig).asInstanceOf[MutableScenario] scenario.setNetwork(networkCoordinator.network) diff --git a/src/main/scala/beam/sim/BeamMobsim.scala b/src/main/scala/beam/sim/BeamMobsim.scala index 5fa648a791f..ebf02789191 100755 --- a/src/main/scala/beam/sim/BeamMobsim.scala +++ b/src/main/scala/beam/sim/BeamMobsim.scala @@ -20,9 +20,6 @@ import beam.agentsim.agents.ridehail.RideHailManager.{ import beam.agentsim.agents.ridehail.{RideHailAgent, RideHailManager, RideHailSurgePricingManager} import beam.agentsim.agents.vehicles.EnergyEconomyAttributes.Powertrain import beam.agentsim.agents.vehicles._ -import beam.agentsim.infrastructure.ParkingManager.ParkingStockAttributes -import beam.agentsim.infrastructure.{ParkingManager, TAZTreeMap, ZonalParkingManager} -import beam.agentsim.scheduler.{BeamAgentScheduler, Trigger} import beam.agentsim.agents.{BeamAgent, InitializeTrigger, Population} import beam.agentsim.infrastructure.ParkingManager.ParkingStockAttributes import beam.agentsim.infrastructure.ZonalParkingManager @@ -40,9 +37,10 @@ import org.matsim.api.core.v01.population.{Activity, Person} import org.matsim.api.core.v01.{Coord, Id, Scenario} import org.matsim.core.api.experimental.events.EventsManager import org.matsim.core.mobsim.framework.Mobsim +import org.matsim.core.scenario.{MutableScenario, ScenarioUtils} import org.matsim.core.utils.misc.Time import org.matsim.households.Household -import org.matsim.vehicles.{Vehicle, VehicleType, VehicleUtils} +import org.matsim.vehicles.VehicleType import scala.collection.JavaConverters._ import scala.collection.mutable @@ -208,12 +206,13 @@ class BeamMobsim @Inject()( private val numRideHailAgents = math.round( beamServices.beamConfig.beam.agentsim.numAgents.toDouble * beamServices.beamConfig.beam.agentsim.agents.rideHail.numDriversAsFractionOfPopulation ) - private val rideHailVehicleType = BeamVehicleUtils - .getVehicleTypeById( - beamServices.beamConfig.beam.agentsim.agents.rideHail.vehicleTypeId, - scenario.getVehicles.getVehicleTypes - ) - .getOrElse(scenario.getVehicles.getVehicleTypes.get(Id.create("1", classOf[VehicleType]))) + private val rideHailVehicleType = + BeamVehicleUtils + .getVehicleTypeById( + beamServices.beamConfig.beam.agentsim.agents.rideHail.vehicleTypeId, + scenario.getVehicles.getVehicleTypes + ) + .getOrElse(scenario.getVehicles.getVehicleTypes.get(Id.create("1", classOf[VehicleType]))) val quadTreeBounds: QuadTreeBounds = getQuadTreeBound( scenario.getPopulation.getPersons @@ -368,8 +367,10 @@ class BeamMobsim @Inject()( Await.result(beamServices.beamRouter ? InitTransit(scheduler, parkingManager), timeout.duration) - if (beamServices.iterationNumber == 0) - new BeamWarmStart(beamServices).init() + if (beamServices.iterationNumber == 0) { + val warmStart = BeamWarmStart(beamServices.beamConfig) + warmStart.warmStartRouterIfNeeded(beamServices.beamRouter) + } log.info(s"Transit schedule has been initialized") diff --git a/src/main/scala/beam/sim/BeamWarmStart.scala b/src/main/scala/beam/sim/BeamWarmStart.scala index 0cb809716c7..08af25df729 100755 --- a/src/main/scala/beam/sim/BeamWarmStart.scala +++ b/src/main/scala/beam/sim/BeamWarmStart.scala @@ -3,19 +3,26 @@ package beam.sim import java.io.File import java.nio.file.{Files, Paths} +import akka.actor.ActorRef import beam.router.BeamRouter.UpdateTravelTime -import beam.router.LinkTravelTimeContainer +import beam.router.{BeamRouter, LinkTravelTimeContainer} +import beam.sim.config.BeamConfig import beam.utils.FileUtils.downloadFile -import beam.utils.UnzipUtility.unzip +import beam.utils.UnzipUtility._ import com.typesafe.scalalogging.LazyLogging import org.apache.commons.io.FileUtils.getTempDirectoryPath import org.apache.commons.io.FilenameUtils.{getBaseName, getExtension, getName} +import org.matsim.core.config.Config import org.matsim.core.router.util.TravelTime +import org.matsim.core.scenario.{MutableScenario, ScenarioUtils} import scala.compat.java8.StreamConverters._ -class BeamWarmStart(val beamServices: BeamServices) extends LazyLogging { - private val beamConfig = beamServices.beamConfig +object BeamWarmStart { + def apply(beamConfig: BeamConfig): BeamWarmStart = new BeamWarmStart(beamConfig) +} + +class BeamWarmStart(beamConfig: BeamConfig) extends LazyLogging { // beamConfig.beam.warmStart.pathType=PARENT_RUN, ABSOLUTE_PATH private val pathType = beamConfig.beam.warmStart.pathType private val srcPath = beamConfig.beam.warmStart.path @@ -30,13 +37,12 @@ class BeamWarmStart(val beamServices: BeamServices) extends LazyLogging { /** * initialize warm start mode. */ - def init(): Unit = { + def warmStartRouterIfNeeded(beamRouter: ActorRef): Unit = { if (!isWarmMode) return - - getWarmStartPath match { + warmStartPath match { case Some(statsPath) => if (Files.exists(Paths.get(statsPath))) { - beamServices.beamRouter ! UpdateTravelTime(getTravelTime(statsPath)) + beamRouter ! UpdateTravelTime(getTravelTime(statsPath)) logger.info(s"Warm start mode initialized successfully from stats located at $statsPath.") } else { logger.warn( @@ -44,27 +50,46 @@ class BeamWarmStart(val beamServices: BeamServices) extends LazyLogging { ) } case None => + logger.warn( + s"Warm start mode initialization failed - warmStartPath is None" + ) } } - private def getWarmStartPath: Option[String] = { - pathType match { - case "PARENT_RUN" => - getWarmStartPath(getParentRunPath) - - case "ABSOLUTE_PATH" => - Files - .walk(Paths.get(srcPath)) - .toScala[Stream] - .map(_.toString) - .find(_.endsWith(".linkstats.csv.gz")) - - case _ => - logger.warn(s"Warm start mode initialization failed, not a valid path type ( $pathType )") - None + def warmStartPopulationIfNeeded(matsimConfig: Config): Unit = { + if (isWarmMode) { + populationFilePath.foreach { file => + matsimConfig.plans().setInputFile(file) + logger.info("Warm start population initialized successfully from file located at {}", file) + } } } + lazy val populationFilePath: Option[String] = + if (isWarmMode) { + val path = pathType match { + case "PARENT_RUN" => parentRunPath + case "ABSOLUTE_PATH" => srcPath + } + Some(loadPopulation(path, populationFile(path))) + } else None + + private lazy val warmStartPath: Option[String] = pathType match { + case "PARENT_RUN" => + getWarmStartPath(parentRunPath) + + case "ABSOLUTE_PATH" => + Files + .walk(Paths.get(srcPath)) + .toScala[Stream] + .map(_.toString) + .find(_.endsWith(".linkstats.csv.gz")) + + case _ => + logger.warn(s"Warm start mode initialization failed, not a valid path type ( $pathType )") + None + } + private def getWarmStartPath(runPath: String) = { val iterOption = Files .walk(Paths.get(runPath)) @@ -93,7 +118,7 @@ class BeamWarmStart(val beamServices: BeamServices) extends LazyLogging { } } - private def getParentRunPath = { + private lazy val parentRunPath = { if (isZipArchive(srcPath)) { var archivePath = srcPath if (isOutputBucketUrl(srcPath)) { @@ -102,12 +127,20 @@ class BeamWarmStart(val beamServices: BeamServices) extends LazyLogging { } val runPath = Paths.get(getTempDirectoryPath, getBaseName(srcPath)).toString unzip(archivePath, runPath, false) - runPath + Paths.get(runPath, getBaseName(srcPath)).toString } else { srcPath } } + private def populationFile(runPath: String): String = Paths.get(runPath, "output_plans.xml.gz").toString + + private def loadPopulation(runPath: String, populationFile: String): String = { + val plansPath = Paths.get(runPath, "output_plans.xml") + unGunzipFile(populationFile, plansPath.toString, false) + plansPath.toString + } + private def isOutputBucketUrl(source: String): Boolean = { assert(source != null) source.startsWith("https://s3.us-east-2.amazonaws.com/beam-outputs/") diff --git a/src/main/scala/beam/sim/config/BeamConfig.scala b/src/main/scala/beam/sim/config/BeamConfig.scala index 757a062e1ad..f07b5c4b3d4 100755 --- a/src/main/scala/beam/sim/config/BeamConfig.scala +++ b/src/main/scala/beam/sim/config/BeamConfig.scala @@ -1,4 +1,4 @@ -// generated by tscfg 0.9.4 on Thu Sep 27 00:23:31 IST 2018 +// generated by tscfg 0.9.4 on Tue Oct 02 08:29:01 PDT 2018 // source: src/main/resources/beam-template.conf package beam.sim.config @@ -392,7 +392,7 @@ object BeamConfig { object Mode { def apply(c: com.typesafe.config.Config): BeamConfig.Beam.Calibration.Mode = { BeamConfig.Beam.Calibration.Mode( - benchmarkFileLoc = if(c.hasPathOrNull("benchmarkFileLoc")) c.getString("benchmarkFileLoc") else "/test/input/beamville/sf-light-calibration/benchmark.csv" + benchmarkFileLoc = if(c.hasPathOrNull("benchmarkFileLoc")) c.getString("benchmarkFileLoc") else "" ) } } @@ -538,7 +538,7 @@ object BeamConfig { def apply(c: com.typesafe.config.Config): BeamConfig.Beam.Outputs = { BeamConfig.Beam.Outputs( addTimestampToOutputDirectory = !c.hasPathOrNull("addTimestampToOutputDirectory") || c.getBoolean("addTimestampToOutputDirectory"), - baseOutputDirectory = if(c.hasPathOrNull("baseOutputDirectory")) c.getString("baseOutputDirectory") else "output", + baseOutputDirectory = if(c.hasPathOrNull("baseOutputDirectory")) c.getString("baseOutputDirectory") else "/Users/critter/Documents/beam/beam-output/", events = BeamConfig.Beam.Outputs.Events(if(c.hasPathOrNull("events")) c.getConfig("events") else com.typesafe.config.ConfigFactory.parseString("events{}")), stats = BeamConfig.Beam.Outputs.Stats(if(c.hasPathOrNull("stats")) c.getConfig("stats") else com.typesafe.config.ConfigFactory.parseString("stats{}")), writeEventsInterval = if(c.hasPathOrNull("writeEventsInterval")) c.getInt("writeEventsInterval") else 1, @@ -548,16 +548,18 @@ object BeamConfig { } case class Physsim( - flowCapacityFactor : scala.Double, - jdeqsim : BeamConfig.Beam.Physsim.Jdeqsim, - linkStatsBinSize : scala.Int, - linkStatsWriteInterval : scala.Int, - ptSampleSize : scala.Double, - skipPhysSim : scala.Boolean, - storageCapacityFactor : scala.Double, - writeEventsInterval : scala.Int, - writeMATSimNetwork : scala.Boolean, - writePlansInterval : scala.Int + eventsForFullVersionOfVia : scala.Boolean, + flowCapacityFactor : scala.Double, + jdeqsim : BeamConfig.Beam.Physsim.Jdeqsim, + linkStatsBinSize : scala.Int, + linkStatsWriteInterval : scala.Int, + ptSampleSize : scala.Double, + quick_fix_minCarSpeedInMetersPerSecond : scala.Double, + skipPhysSim : scala.Boolean, + storageCapacityFactor : scala.Double, + writeEventsInterval : scala.Int, + writeMATSimNetwork : scala.Boolean, + writePlansInterval : scala.Int ) object Physsim { case class Jdeqsim( @@ -584,16 +586,18 @@ object BeamConfig { def apply(c: com.typesafe.config.Config): BeamConfig.Beam.Physsim = { BeamConfig.Beam.Physsim( - flowCapacityFactor = if(c.hasPathOrNull("flowCapacityFactor")) c.getDouble("flowCapacityFactor") else 1.0, - jdeqsim = BeamConfig.Beam.Physsim.Jdeqsim(if(c.hasPathOrNull("jdeqsim")) c.getConfig("jdeqsim") else com.typesafe.config.ConfigFactory.parseString("jdeqsim{}")), - linkStatsBinSize = if(c.hasPathOrNull("linkStatsBinSize")) c.getInt("linkStatsBinSize") else 3600, - linkStatsWriteInterval = if(c.hasPathOrNull("linkStatsWriteInterval")) c.getInt("linkStatsWriteInterval") else 1, - ptSampleSize = if(c.hasPathOrNull("ptSampleSize")) c.getDouble("ptSampleSize") else 1.0, - skipPhysSim = c.hasPathOrNull("skipPhysSim") && c.getBoolean("skipPhysSim"), - storageCapacityFactor = if(c.hasPathOrNull("storageCapacityFactor")) c.getDouble("storageCapacityFactor") else 1.0, - writeEventsInterval = if(c.hasPathOrNull("writeEventsInterval")) c.getInt("writeEventsInterval") else 0, - writeMATSimNetwork = c.hasPathOrNull("writeMATSimNetwork") && c.getBoolean("writeMATSimNetwork"), - writePlansInterval = if(c.hasPathOrNull("writePlansInterval")) c.getInt("writePlansInterval") else 0 + eventsForFullVersionOfVia = !c.hasPathOrNull("eventsForFullVersionOfVia") || c.getBoolean("eventsForFullVersionOfVia"), + flowCapacityFactor = if(c.hasPathOrNull("flowCapacityFactor")) c.getDouble("flowCapacityFactor") else 1.0, + jdeqsim = BeamConfig.Beam.Physsim.Jdeqsim(if(c.hasPathOrNull("jdeqsim")) c.getConfig("jdeqsim") else com.typesafe.config.ConfigFactory.parseString("jdeqsim{}")), + linkStatsBinSize = if(c.hasPathOrNull("linkStatsBinSize")) c.getInt("linkStatsBinSize") else 3600, + linkStatsWriteInterval = if(c.hasPathOrNull("linkStatsWriteInterval")) c.getInt("linkStatsWriteInterval") else 1, + ptSampleSize = if(c.hasPathOrNull("ptSampleSize")) c.getDouble("ptSampleSize") else 1.0, + quick_fix_minCarSpeedInMetersPerSecond = if(c.hasPathOrNull("quick_fix_minCarSpeedInMetersPerSecond")) c.getDouble("quick_fix_minCarSpeedInMetersPerSecond") else 0.5, + skipPhysSim = c.hasPathOrNull("skipPhysSim") && c.getBoolean("skipPhysSim"), + storageCapacityFactor = if(c.hasPathOrNull("storageCapacityFactor")) c.getDouble("storageCapacityFactor") else 1.0, + writeEventsInterval = if(c.hasPathOrNull("writeEventsInterval")) c.getInt("writeEventsInterval") else 0, + writeMATSimNetwork = c.hasPathOrNull("writeMATSimNetwork") && c.getBoolean("writeMATSimNetwork"), + writePlansInterval = if(c.hasPathOrNull("writePlansInterval")) c.getInt("writePlansInterval") else 0 ) } } @@ -615,7 +619,7 @@ object BeamConfig { BeamConfig.Beam.Routing.Gtfs( crs = if(c.hasPathOrNull("crs")) c.getString("crs") else "epsg:26910", operatorsFile = if(c.hasPathOrNull("operatorsFile")) c.getString("operatorsFile") else "src/main/resources/GTFSOperators.csv", - outputDir = if(c.hasPathOrNull("outputDir")) c.getString("outputDir") else "output/gtfs" + outputDir = if(c.hasPathOrNull("outputDir")) c.getString("outputDir") else "/Users/critter/Documents/beam/beam-output//gtfs" ) } } @@ -686,7 +690,7 @@ object BeamConfig { def apply(c: com.typesafe.config.Config): BeamConfig.Beam.WarmStart = { BeamConfig.Beam.WarmStart( enabled = c.hasPathOrNull("enabled") && c.getBoolean("enabled"), - path = if(c.hasPathOrNull("path")) c.getString("path") else "output", + path = if(c.hasPathOrNull("path")) c.getString("path") else "/Users/critter/Documents/beam/beam-output/", pathType = if(c.hasPathOrNull("pathType")) c.getString("pathType") else "PARENT_RUN" ) } @@ -810,7 +814,7 @@ object BeamConfig { firstIteration = if(c.hasPathOrNull("firstIteration")) c.getInt("firstIteration") else 0, lastIteration = if(c.hasPathOrNull("lastIteration")) c.getInt("lastIteration") else 0, mobsim = if(c.hasPathOrNull("mobsim")) c.getString("mobsim") else "metasim", - outputDirectory = if(c.hasPathOrNull("outputDirectory")) c.getString("outputDirectory") else "output/pt-tutorial", + outputDirectory = if(c.hasPathOrNull("outputDirectory")) c.getString("outputDirectory") else "/Users/critter/Documents/beam/beam-output//pt-tutorial", overwriteFiles = if(c.hasPathOrNull("overwriteFiles")) c.getString("overwriteFiles") else "overwriteExistingFiles" ) } diff --git a/src/main/scala/beam/utils/plansampling/AvailableModeUtils.scala b/src/main/scala/beam/utils/plansampling/AvailableModeUtils.scala index caca7c7a396..8019846aeed 100755 --- a/src/main/scala/beam/utils/plansampling/AvailableModeUtils.scala +++ b/src/main/scala/beam/utils/plansampling/AvailableModeUtils.scala @@ -4,6 +4,7 @@ import java.util import beam.agentsim.agents.household.HouseholdActor.AttributesOfIndividual import beam.router.Modes.BeamMode +import beam.router.Modes.BeamMode.{DRIVE_TRANSIT, RIDE_HAIL, RIDE_HAIL_TRANSIT, WALK, WALK_TRANSIT} import org.matsim.api.core.v01.Id import org.matsim.api.core.v01.population.{Person, Plan} import org.matsim.core.population.algorithms.PermissibleModesCalculator @@ -28,10 +29,15 @@ object AvailableModeUtils { } def availableModesForPerson(person: Person): Seq[BeamMode] = { - person.getCustomAttributes + val availModes = person.getCustomAttributes .get("beam-attributes") .asInstanceOf[AttributesOfIndividual] - .availableModes + .availableModes :+ WALK :+ WALK_TRANSIT :+ DRIVE_TRANSIT + if(availModes.contains(RIDE_HAIL)){ + availModes :+ RIDE_HAIL_TRANSIT + }else{ + availModes + } } def isModeAvailableForPerson[T <: BeamMode]( diff --git a/src/test/java/beam/analysis/physsim/PhyssimCalcLinkSpeedDistributionStatsTest.java b/src/test/java/beam/analysis/physsim/PhyssimCalcLinkSpeedDistributionStatsTest.java new file mode 100755 index 00000000000..c86c76e2525 --- /dev/null +++ b/src/test/java/beam/analysis/physsim/PhyssimCalcLinkSpeedDistributionStatsTest.java @@ -0,0 +1,106 @@ +package beam.analysis.physsim; + +import org.junit.BeforeClass; +import org.junit.Test; +import org.matsim.api.core.v01.Scenario; +import org.matsim.api.core.v01.network.Link; +import org.matsim.api.core.v01.network.Network; +import org.matsim.core.api.experimental.events.EventsManager; +import org.matsim.core.config.Config; +import org.matsim.core.config.ConfigUtils; +import org.matsim.core.config.groups.TravelTimeCalculatorConfigGroup; +import org.matsim.core.events.EventsUtils; +import org.matsim.core.events.MatsimEventsReader; +import org.matsim.core.network.io.MatsimNetworkReader; +import org.matsim.core.scenario.ScenarioUtils; +import org.matsim.core.trafficmonitoring.TravelTimeCalculator; + +import java.nio.file.Paths; +import java.util.*; +import java.util.stream.Stream; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotEquals; +import static org.junit.Assert.assertTrue; + +public class PhyssimCalcLinkSpeedDistributionStatsTest { + + private static final String BASE_PATH = Paths.get(".").toAbsolutePath().toString(); + private static final String EVENTS_FILE_PATH = BASE_PATH + "/test/input/equil-square/test-data/physSimEvents-relative-speeds.xml"; + private static final String NETWORK_FILE_PATH = BASE_PATH + "/test/input/equil-square/test-data/physSimNetwork-relative-speeds.xml"; + + private static PhyssimCalcLinkSpeedDistributionStats physsimCalcLinkSpeedDistributionStats; + private static TravelTimeCalculator travelTimeCalculator; + private static Network network; + private int binCount = 10; + + @BeforeClass + public static void createDummySimWithXML() { + + Config config = ConfigUtils.createConfig(); + Scenario scenario = ScenarioUtils.createScenario(ConfigUtils.createConfig()); + network = scenario.getNetwork(); + MatsimNetworkReader matsimNetworkReader = new MatsimNetworkReader(network); + matsimNetworkReader.readFile(NETWORK_FILE_PATH); + + TravelTimeCalculatorConfigGroup defaultTravelTimeCalculator = config.travelTimeCalculator(); + travelTimeCalculator = new TravelTimeCalculator(network, defaultTravelTimeCalculator); + EventsManager eventsManager = EventsUtils.createEventsManager(); + eventsManager.addHandler(travelTimeCalculator); + + physsimCalcLinkSpeedDistributionStats = new PhyssimCalcLinkSpeedDistributionStats(network, null, null); + + physsimCalcLinkSpeedDistributionStats.notifyIterationStarts(eventsManager); + + MatsimEventsReader matsimEventsReader = new MatsimEventsReader(eventsManager); + matsimEventsReader.readFile(EVENTS_FILE_PATH); + physsimCalcLinkSpeedDistributionStats.notifyIterationEnds(0, travelTimeCalculator); + } + + private int getMaxSpeed(int binCount, Network network) { + return Stream.iterate(0, x -> x) + .limit(binCount) + .flatMap(bin -> network.getLinks().values() + .stream() + .map(link -> link.getFreespeed(bin * 3600)) + .map(fs -> (int) Math.round(fs))) + .max(Comparator.comparing(Integer::valueOf)).orElse(0); + } + + private Stream getDistinctSpeeds(int binCount, Network network) { + return Stream.iterate(0, x -> x) + .limit(binCount) + .flatMap(bin -> network.getLinks().values() + .stream() + .map(link -> link.getFreespeed(bin * 3600)) + .map(fs -> (int) Math.round(fs))) + .distinct(); + } + + @Test + public void shouldCheckIfSumOfFrequenciesPerBinEqualsTotalNumberOfLinks() { + int linksCount = network.getLinks().size(); + Map freeFlowSpeedFrequencies = physsimCalcLinkSpeedDistributionStats.generateInputDataForFreeFlowSpeedGraph(binCount,network); + int sumOfFrequencies = freeFlowSpeedFrequencies.values().stream().reduce((r, s) -> r + s).orElse(0); + assertEquals(sumOfFrequencies/binCount, linksCount); + } + + @Test + public void shouldHaveAtleastOneLinkWithMaximumSpeed() { + Map freeFlowSpeedFrequencies = physsimCalcLinkSpeedDistributionStats.generateInputDataForFreeFlowSpeedGraph(binCount,network); + int maxSpeed = this.getMaxSpeed(binCount, network); + Integer linksWithMaxSpeed = freeFlowSpeedFrequencies.getOrDefault(maxSpeed, 0);//.stream().filter(f -> f.equals(Integer.valueOf(maxSpeed))).count(); + assertTrue(linksWithMaxSpeed > 0); + } + + @Test + public void shouldHaveDomainAxesEqualsToDistinctFreeFlowSpeeds() { + Map freeFlowSpeedFrequencies = physsimCalcLinkSpeedDistributionStats.generateInputDataForFreeFlowSpeedGraph(binCount,network); + Stream distinctSpeeds = this.getDistinctSpeeds(binCount, network).sorted(); + Iterator iter1 = distinctSpeeds.iterator(), iter2 = freeFlowSpeedFrequencies.keySet().stream().sorted().iterator(); + while(iter1.hasNext() && iter2.hasNext()) + assertEquals(iter1.next(), iter2.next()); + assert !iter1.hasNext() && !iter2.hasNext(); + } + +} diff --git a/src/test/java/beam/analysis/physsim/PhyssimCalcLinkSpeedStatsTest.java b/src/test/java/beam/analysis/physsim/PhyssimCalcLinkSpeedStatsTest.java new file mode 100755 index 00000000000..b7c4f5ba071 --- /dev/null +++ b/src/test/java/beam/analysis/physsim/PhyssimCalcLinkSpeedStatsTest.java @@ -0,0 +1,77 @@ +package beam.analysis.physsim; + +import org.junit.BeforeClass; +import org.junit.Test; +import org.matsim.api.core.v01.Scenario; +import org.matsim.api.core.v01.network.Network; +import org.matsim.core.api.experimental.events.EventsManager; +import org.matsim.core.config.Config; +import org.matsim.core.config.ConfigUtils; +import org.matsim.core.config.groups.TravelTimeCalculatorConfigGroup; +import org.matsim.core.events.EventsUtils; +import org.matsim.core.events.MatsimEventsReader; +import org.matsim.core.network.io.MatsimNetworkReader; +import org.matsim.core.scenario.ScenarioUtils; +import org.matsim.core.trafficmonitoring.TravelTimeCalculator; + +import java.nio.file.Paths; +import java.util.Arrays; +import java.util.List; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotEquals; + +public class PhyssimCalcLinkSpeedStatsTest { + + private static final String BASE_PATH = Paths.get(".").toAbsolutePath().toString(); + private static final String EVENTS_FILE_PATH = BASE_PATH + "/test/input/equil-square/test-data/physSimEvents-relative-speeds.xml"; + private static final String NETWORK_FILE_PATH = BASE_PATH + "/test/input/equil-square/test-data/physSimNetwork-relative-speeds.xml"; + + private static PhyssimCalcLinkSpeedStats physsimCalcLinkSpeedStats; + private static TravelTimeCalculator travelTimeCalculator; + + @BeforeClass + public static void createDummySimWithXML() { + + Config config = ConfigUtils.createConfig(); + Scenario scenario = ScenarioUtils.createScenario(ConfigUtils.createConfig()); + Network network = scenario.getNetwork(); + MatsimNetworkReader matsimNetworkReader = new MatsimNetworkReader(network); + matsimNetworkReader.readFile(NETWORK_FILE_PATH); + + TravelTimeCalculatorConfigGroup defaultTravelTimeCalculator = config.travelTimeCalculator(); + travelTimeCalculator = new TravelTimeCalculator(network, defaultTravelTimeCalculator); + EventsManager eventsManager = EventsUtils.createEventsManager(); + eventsManager.addHandler(travelTimeCalculator); + + physsimCalcLinkSpeedStats = new PhyssimCalcLinkSpeedStats(network, null, null); + + physsimCalcLinkSpeedStats.notifyIterationStarts(eventsManager); + + MatsimEventsReader matsimEventsReader = new MatsimEventsReader(eventsManager); + matsimEventsReader.readFile(EVENTS_FILE_PATH); + physsimCalcLinkSpeedStats.notifyIterationEnds(0, travelTimeCalculator); + } + + @Test + public void shouldReturnAverageRelativeSpeedPercentageOfSpecificBin() { + double expectedResult = 96.0; + double actualResult = physsimCalcLinkSpeedStats.getAverageSpeedPercentageOfBin(23,travelTimeCalculator); + assertEquals(expectedResult, Math.ceil(actualResult),0); + } + + @Test + public void shouldNotContainAverageRelativeSpeedPercentageOfHundredForAllBins() { + double[] actualResult = physsimCalcLinkSpeedStats.getAverageSpeedPercentagesOfAllBins(travelTimeCalculator); + long nonHundredPercentages = Arrays.stream(actualResult).filter(f -> f != 100.0).count(); + assertNotEquals(nonHundredPercentages,0); + } + + @Test + public void shouldHaveDomainAxisRangeEqualToNumberOfBins() { + int binCount = physsimCalcLinkSpeedStats.getNumberOfBins(); + double[] actualResult = physsimCalcLinkSpeedStats.getAverageSpeedPercentagesOfAllBins(travelTimeCalculator); + assertEquals(binCount,actualResult.length); + } + +} diff --git a/src/test/java/beam/analysis/physsim/PhyssimCalcLinkStatsTest.java b/src/test/java/beam/analysis/physsim/PhyssimCalcLinkStatsTest.java index d9387f595d6..eddad3c53bb 100755 --- a/src/test/java/beam/analysis/physsim/PhyssimCalcLinkStatsTest.java +++ b/src/test/java/beam/analysis/physsim/PhyssimCalcLinkStatsTest.java @@ -1,5 +1,8 @@ package beam.analysis.physsim; +import beam.sim.config.BeamConfig; +import beam.utils.TestConfigUtils; +import com.typesafe.config.ConfigValueFactory; import org.junit.BeforeClass; import org.junit.Test; import org.matsim.api.core.v01.Scenario; @@ -13,6 +16,7 @@ import org.matsim.core.network.io.MatsimNetworkReader; import org.matsim.core.scenario.ScenarioUtils; import org.matsim.core.trafficmonitoring.TravelTimeCalculator; +import beam.utils.TestConfigUtils; import java.nio.file.Paths; import java.util.List; @@ -41,7 +45,9 @@ public static void createDummySimWithXML() { EventsManager eventsManager = EventsUtils.createEventsManager(); eventsManager.addHandler(travelTimeCalculator); - physsimCalcLinkStats = new PhyssimCalcLinkStats(network, null, null); + physsimCalcLinkStats = new PhyssimCalcLinkStats(network, null, BeamConfig.apply(TestConfigUtils.testConfig("test/input/equil-square/equil-0.001k.conf").withValue("beam.physsim.quick_fix_minCarSpeedInMetersPerSecond", ConfigValueFactory.fromAnyRef(0.0)))); + + //physsimCalcLinkStats = new PhyssimCalcLinkStats(network, null, null); physsimCalcLinkStats.notifyIterationStarts(eventsManager); diff --git a/src/test/scala/beam/analisis/physsim/BeamCalcLinkStatsSpec.scala b/src/test/scala/beam/analisis/physsim/BeamCalcLinkStatsSpec.scala index a4fba59e30e..bf1c705a950 100755 --- a/src/test/scala/beam/analisis/physsim/BeamCalcLinkStatsSpec.scala +++ b/src/test/scala/beam/analisis/physsim/BeamCalcLinkStatsSpec.scala @@ -53,7 +53,7 @@ class BeamCalcLinkStatsSpec extends WordSpecLike with Matchers with BeforeAndAft val events = EventsUtils.createEventsManager() events.addHandler(travelTimeCalculator) - beamCalcLinkStats = new BeamCalcLinkStats(network) + beamCalcLinkStats = new BeamCalcLinkStats(network, 0.5) beamCalcLinkStats.reset() val volumes = new VolumesAnalyzer(3600, 24 * 3600 - 1, network) events.addHandler(volumes) diff --git a/src/test/scala/beam/integration/ParkingSpec.scala b/src/test/scala/beam/integration/ParkingSpec.scala index 960c840ac19..7bae1954454 100755 --- a/src/test/scala/beam/integration/ParkingSpec.scala +++ b/src/test/scala/beam/integration/ParkingSpec.scala @@ -236,7 +236,7 @@ class ParkingSpec .sum should be > expensiveModeChoiceCarCount.takeRight(5).sum } - "empty parking access should reduce driving" in { + "empty parking access should reduce driving" ignore { val emptyModeChoiceCarCount = emptyEvents.map(filterForCarMode) val defaultModeChoiceCarCount = defaultEvents.map(filterForCarMode) diff --git a/src/test/scala/beam/router/TimeDependentRoutingSpec.scala b/src/test/scala/beam/router/TimeDependentRoutingSpec.scala index e8c56c3993b..6a1b04e9d9e 100755 --- a/src/test/scala/beam/router/TimeDependentRoutingSpec.scala +++ b/src/test/scala/beam/router/TimeDependentRoutingSpec.scala @@ -241,7 +241,7 @@ class TimeDependentRoutingSpec ) carOption = expectMsgType[RoutingResponse].itineraries .find(_.tripClassifier == CAR) - .get + .getOrElse(carOption) } assert(scala.math.abs(gap) < 75) // isn't exactly 0, probably rounding errors? diff --git a/src/test/scala/beam/router/WarmStartRoutingSpec.scala b/src/test/scala/beam/router/WarmStartRoutingSpec.scala index db4a21a687d..4aee92591a7 100755 --- a/src/test/scala/beam/router/WarmStartRoutingSpec.scala +++ b/src/test/scala/beam/router/WarmStartRoutingSpec.scala @@ -26,7 +26,7 @@ import org.matsim.core.scenario.ScenarioUtils import org.mockito.ArgumentMatchers.any import org.mockito.Mockito.when import org.scalatest.mockito.MockitoSugar -import org.scalatest.{BeforeAndAfterAll, Ignore, Matchers, WordSpecLike} +import org.scalatest.{BeforeAndAfterAll, Matchers, WordSpecLike} import scala.concurrent.duration._ import scala.language.postfixOps @@ -125,12 +125,12 @@ class WarmStartRoutingSpec ) ) ) - val response = expectMsgType[RoutingResponse] + var response = expectMsgType[RoutingResponse] assert(response.itineraries.exists(_.tripClassifier == CAR)) val carOption = response.itineraries.find(_.tripClassifier == CAR).get assert(carOption.totalTravelTimeInSecs == 76) - new BeamWarmStart(services).init() + new BeamWarmStart(services.beamConfig).warmStartRouterIfNeeded(services.beamRouter) router ! RoutingRequest( origin, destination, @@ -145,9 +145,9 @@ class WarmStartRoutingSpec ) ) ) - val response2 = expectMsgType[RoutingResponse] - assert(response2.itineraries.exists(_.tripClassifier == CAR)) - val carOption2 = response2.itineraries.find(_.tripClassifier == CAR).get + response = expectMsgType[RoutingResponse] + assert(response.itineraries.exists(_.tripClassifier == CAR)) + val carOption2 = response.itineraries.find(_.tripClassifier == CAR).get assert(carOption2.totalTravelTimeInSecs == 55) } } diff --git a/test.csv b/test.csv new file mode 100644 index 00000000000..2784ef470dd --- /dev/null +++ b/test.csv @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:aecbe6c534604ecf58a4da7885a5269d6f31e9e5e79dcd37ef1407e6c50b64da +size 28 diff --git a/test/input/beamville/beam.conf b/test/input/beamville/beam.conf index 0ff5651f769..8c33e0fb404 100755 --- a/test/input/beamville/beam.conf +++ b/test/input/beamville/beam.conf @@ -115,6 +115,8 @@ beam.agentsim.agents.vehicles.beamVehiclesFile = ${beam.inputDirectory}"/vehicle # Population Adjustmnet (DEFAULT_ADJUSTMENT | PERCENTAGE_ADJUSTMENT) beam.agentsim.populationAdjustment="DEFAULT_ADJUSTMENT" + +beam.physsim.quick_fix_minCarSpeedInMetersPerSecond = 0.0 ################################################################## # Debugging ################################################################## diff --git a/test/input/beamville/lccm-simple.csv b/test/input/beamville/lccm-simple.csv new file mode 100644 index 00000000000..724fca8cc98 --- /dev/null +++ b/test/input/beamville/lccm-simple.csv @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f7f903b2dc5ae14f88bcfbdc6f20a1519a289b83db980739f75487624ef001ad +size 1888 diff --git a/test/input/equil-square/equil-0.001k.conf b/test/input/equil-square/equil-0.001k.conf index b7d5be7aacc..96078a03af6 100755 --- a/test/input/equil-square/equil-0.001k.conf +++ b/test/input/equil-square/equil-0.001k.conf @@ -40,6 +40,11 @@ beam.physsim.writeMATSimNetwork = true beam.debug.debugEnabled = true beam.debug.stuckAgentDetection.enabled = false +beam.debug.stuckAgentDetection { + enabled = false + thresholds = [] +} + ################################################################## # OUTPUTS ################################################################## diff --git a/test/input/sf-light/sample/1k/vehicleTypes.csv b/test/input/sf-light/sample/1k/vehicleTypes.csv new file mode 100644 index 00000000000..c64c4e048f0 --- /dev/null +++ b/test/input/sf-light/sample/1k/vehicleTypes.csv @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:048eccb83fc97c6e8098971c98ee60d80a6ae31433996283ccab89bc6a72dde3 +size 3536 diff --git a/test/input/sf-light/sf-light-25k.conf b/test/input/sf-light/sf-light-25k.conf index a48cf07952a..f6ca5563cf9 100755 --- a/test/input/sf-light/sf-light-25k.conf +++ b/test/input/sf-light/sf-light-25k.conf @@ -33,8 +33,8 @@ beam.agentsim.agents.modalBehaviors.lccm.paramFile = ${beam.inputDirectory}"/lcc beam.agentsim.agents.vehicles.bicycles.useBikes=false #BeamVehicles Params beam.agentsim.agents.vehicles.beamFuelTypesFile = ${beam.inputDirectory}"/beamFuelTypes.csv" -beam.agentsim.agents.vehicles.beamVehicleTypesFile = ${beam.inputDirectory}"/sample/25k/vehicleTypes.csv.gz" -beam.agentsim.agents.vehicles.beamVehiclesFile = ${beam.inputDirectory}"/sample/25k/vehicles.csv.gz" +beam.agentsim.agents.vehicles.beamVehicleTypesFile = ${beam.inputDirectory}"/sample/25k/vehicleTypes.csv" +beam.agentsim.agents.vehicles.beamVehiclesFile = ${beam.inputDirectory}"/sample/25k/vehicles.csv" #DrivingCostDefaults Params beam.agentsim.agents.drivingCost.defaultLitersPerMeter = 0.0001069 beam.agentsim.agents.drivingCost.defaultPricePerGallon = 3.115 diff --git a/test/input/sf-light/sf-light.conf b/test/input/sf-light/sf-light.conf index cb56f1a3f0e..ecd114d4a07 100644 --- a/test/input/sf-light/sf-light.conf +++ b/test/input/sf-light/sf-light.conf @@ -22,12 +22,12 @@ beam.agentsim.agents.modalBehaviors.defaultValueOfTime = 18 beam.agentsim.agents.modalBehaviors.mulitnomialLogit.params.cost = -1.0 beam.agentsim.agents.modalBehaviors.mulitnomialLogit.params.time = -0.0047 beam.agentsim.agents.modalBehaviors.mulitnomialLogit.params.transfer = -1.4 -beam.agentsim.agents.modalBehaviors.mulitnomialLogit.params.car_intercept = 0.0 -beam.agentsim.agents.modalBehaviors.mulitnomialLogit.params.walk_transit_intercept = 1.0 -beam.agentsim.agents.modalBehaviors.mulitnomialLogit.params.drive_transit_intercept = 1.0 -beam.agentsim.agents.modalBehaviors.mulitnomialLogit.params.ride_hail_transit_intercept = 5.0 -beam.agentsim.agents.modalBehaviors.mulitnomialLogit.params.ride_hail_intercept = -4.0 -beam.agentsim.agents.modalBehaviors.mulitnomialLogit.params.walk_intercept = 0.0 +beam.agentsim.agents.modalBehaviors.mulitnomialLogit.params.car_intercept = 1.0 +beam.agentsim.agents.modalBehaviors.mulitnomialLogit.params.walk_transit_intercept = -1.81 +beam.agentsim.agents.modalBehaviors.mulitnomialLogit.params.drive_transit_intercept = -2.11 +beam.agentsim.agents.modalBehaviors.mulitnomialLogit.params.ride_hail_transit_intercept = 2.44 +beam.agentsim.agents.modalBehaviors.mulitnomialLogit.params.ride_hail_intercept = 1.26 +beam.agentsim.agents.modalBehaviors.mulitnomialLogit.params.walk_intercept = -5.39 beam.agentsim.agents.modalBehaviors.mulitnomialLogit.params.bike_intercept = 0.0 beam.agentsim.agents.modalBehaviors.lccm.paramFile = ${beam.inputDirectory}"/lccm-long.csv" # Use bikes? @@ -41,6 +41,7 @@ beam.agentsim.agents.drivingCost.defaultLitersPerMeter = 0.0001069 beam.agentsim.agents.drivingCost.defaultPricePerGallon = 3.115 #TAZ params beam.agentsim.taz.file=${beam.inputDirectory}"/taz-centers.csv.gz" +beam.agentsim.taz.parking = ${beam.inputDirectory}"/taz-parking.csv.gz" # Ride Hailing Params beam.agentsim.agents.rideHail.numDriversAsFractionOfPopulation=0.05 beam.agentsim.agents.rideHail.defaultCostPerMile=1.25 @@ -241,33 +242,15 @@ matsim.modules { } strategy { maxAgentPlanMemorySize = 6 - planSelectorForRemoval = "tryToKeepOneOfEachClass" - ModuleProbability_1 = 0.3 + ModuleProbability_1 = 0.8 Module_1 = "SelectExpBeta" - ModuleProbability_2 = 0.7 - Module_2 = "GrabExperiencedPlan" - ModuleProbability_3 = 0.1 Module_3 = "ClearRoutes" - ModuleProbability_4 = 0.2 + ModuleProbability_4 = 0.1 Module_4 = "ClearModes" - -# ModuleProbability_3 = 0.5 -# Module_3 = "SwitchModalityStyle" -# ModuleDisableAfterIteration_3 = 20 - - #ModuleProbability_4 = 0.2 - #Module_4 = "UtilityBasedModeChoice" - #ModuleDisableAfterIteration_4 = 45 - - # ModuleProbability_3 = 0.1 - # Module_3 = "TimeAllocationMutator" - - # ModuleProbability_4 = 0.1 - # Module_4 = "ChangeTripMode" } parallelEventHandling { #Estimated number of events during mobsim run. An optional optimization hint for the framework. diff --git a/test/input/sf-light/taz-centers.csv b/test/input/sf-light/taz-centers.csv new file mode 100755 index 00000000000..6cb6166ccbc --- /dev/null +++ b/test/input/sf-light/taz-centers.csv @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e38dd6bb9855090c0b0625824bb7e1534c37c67d9f3354ef90399e941e481020 +size 54268 diff --git a/test/input/sf-light/taz-parking.csv b/test/input/sf-light/taz-parking.csv new file mode 100644 index 00000000000..c0c9a2c1ab4 --- /dev/null +++ b/test/input/sf-light/taz-parking.csv @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b24e88b60ffe3872b47350bd7b6a7d13ddedefbbf8c4f91e1f7f5aff8ecb572d +size 1455916