Skip to content

Commit

Permalink
Merge pull request #1827 from LBNL-UCB-STI/art/merge-develope-to-mast…
Browse files Browse the repository at this point in the history
…er-3

Merge develope to master
  • Loading branch information
REASY committed May 19, 2019
2 parents 58011e3 + bd8426b commit 7255d94
Show file tree
Hide file tree
Showing 131 changed files with 3,871 additions and 1,942 deletions.
6 changes: 5 additions & 1 deletion .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,10 @@ test/input/siouxfalls/*.csv
/src/main/python/household_generator/.idea/
*Rdata
!/test/input/siouxfalls/r5/siouxareametro-sd-us.zip
osm.mapdb
osm.mapdb.p
network.dat
physsim-network.xml
src/main/scala/beam/sandbox
test/scala/beam/sandbox
test/scala/beam/sandbox
log-path_IS_UNDEFINED/
4 changes: 4 additions & 0 deletions aws/src/main/python/beam_lambda/lambda_function.py
Original file line number Diff line number Diff line change
Expand Up @@ -47,10 +47,13 @@
- cd /home/ubuntu/git/beam
- ln -sf /var/log/cloud-init-output.log ./cloud-init-output.log
- /home/ubuntu/git/glip.sh -i "http://icons.iconarchive.com/icons/uiconstock/socialmedia/32/AWS-icon.png" -a "Run Started" -b "Run Name** $TITLED** \\n Instance ID $(ec2metadata --instance-id) \\n Instance type **$(ec2metadata --instance-type)** \\n Host name **$(ec2metadata --public-hostname)** \\n Web browser **http://$(ec2metadata --public-hostname):8000** \\n Region $REGION \\n Batch $UID \\n Branch **$BRANCH** \\n Commit $COMMIT"
- curl -X POST -H 'Content-type: application/json' --data '{"text":"Run Started \\n Run Name** $TITLED** \\n Instance ID $(ec2metadata --instance-id) \\n Instance type **$(ec2metadata --instance-type)** \\n Host name **$(ec2metadata --public-hostname)** \\n Web browser **http://$(ec2metadata --public-hostname):8000** \\n Region $REGION \\n Batch $UID \\n Branch **$BRANCH** \\n Commit $COMMIT"}' https://hooks.slack.com/services/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
- echo "notification sent..."
- echo '0 * * * * /home/ubuntu/git/glip.sh -i "http://icons.iconarchive.com/icons/uiconstock/socialmedia/32/AWS-icon.png" -a "$(ec2metadata --instance-type) instance $(ec2metadata --instance-id) running..." -b "Batch [$UID] completed and instance of type $(ec2metadata --instance-type) is still running in $REGION since last $(($(($(date +%s) - $(cat /tmp/.starttime))) / 3600)) Hour $(($(($(date +%s) - $(cat /tmp/.starttime))) / 60)) Minute."' > /tmp/glip_notification
- echo '0 * * * * curl -X POST -H "Content-type: application/json" --data '"'"'{"$(ec2metadata --instance-type) instance $(ec2metadata --instance-id) running... \\n Batch [$UID] completed and instance of type $(ec2metadata --instance-type) is still running in $REGION since last $(($(($(date +%s) - $(cat /tmp/.starttime))) / 3600)) Hour $(($(($(date +%s) - $(cat /tmp/.starttime))) / 60)) Minute."}'"'"'' > /tmp/slack_notification
- echo "notification saved..."
- crontab /tmp/glip_notification
- crontab /tmp/slack_notification
- crontab -l
- echo "notification scheduled..."
- git fetch
Expand Down Expand Up @@ -79,6 +82,7 @@
- s3glip="\\n S3 output url ${s3p#","}"
- fi
- /home/ubuntu/git/glip.sh -i "http://icons.iconarchive.com/icons/uiconstock/socialmedia/32/AWS-icon.png" -a "Run Completed" -b "Run Name** $TITLED** \\n Instance ID $(ec2metadata --instance-id) \\n Instance type **$(ec2metadata --instance-type)** \\n Host name **$(ec2metadata --public-hostname)** \\n Web browser **http://$(ec2metadata --public-hostname):8000** \\n Region $REGION \\n Batch $UID \\n Branch **$BRANCH** \\n Commit $COMMIT $s3glip \\n Shutdown in $SHUTDOWN_WAIT minutes"
- curl -X POST -H 'Content-type: application/json' --data '{"text":"Run Completed \\n Run Name** $TITLED** \\n Instance ID $(ec2metadata --instance-id) \\n Instance type **$(ec2metadata --instance-type)** \\n Host name **$(ec2metadata --public-hostname)** \\n Web browser **http://$(ec2metadata --public-hostname):8000** \\n Region $REGION \\n Batch $UID \\n Branch **$BRANCH** \\n Commit $COMMIT $s3glip \\n Shutdown in $SHUTDOWN_WAIT minutes"}' https://hooks.slack.com/services/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
- $END_SCRIPT
- sudo shutdown -h +$SHUTDOWN_WAIT
'''))
Expand Down
33 changes: 26 additions & 7 deletions build.gradle
Original file line number Diff line number Diff line change
Expand Up @@ -62,11 +62,24 @@ sourceSets.main.java.srcDirs = []
sourceSets.test.java.srcDirs = []
sourceSets.test.scala.srcDirs = ["src/test/scala", "src/test/java"]

sourceSets {
main {
resources {
srcDir "src/main/resources"
}
}
test {
resources {
srcDir "src/test/resources"
}
}
}

if (project.hasProperty('env')) {
sourceSets {
main {
resources {
srcDirs "src/main/resources", "test/input/" + project.getProperty('env')
srcDirs "test/input/" + project.getProperty('env')
}
}
}
Expand Down Expand Up @@ -99,7 +112,10 @@ allprojects {
dependencies {

//beam-utilities
compile group: 'com.github.LBNL-UCB-STI', name: 'beam-utilities', version: 'v0.2.1'
compile(group: 'com.github.LBNL-UCB-STI', name: 'beam-utilities', version: 'v0.2.1') {
exclude group: 'com.github.michaz', module: 'r5'
}

////////////////////////////
// Java dependencies
////////////////////////////
Expand Down Expand Up @@ -153,7 +169,7 @@ dependencies {
compile "com.typesafe.scala-logging:scala-logging_${scalaBinaryVersion}:3.9.0"
compile "org.slf4j:log4j-over-slf4j:${slf4jVersion}"

compile(group: 'com.github.LBNL-UCB-STI', name: 'r5', version: 'mz-tolls-SNAPSHOT', changing: true) {
compile(group: 'com.github.michaz', name: 'r5', version: '3ab4fa04') {
exclude group: 'ch.qos.logback', module: 'logback-classic'
exclude group: 'org.slf4j', module: 'slf4j-simple'
}
Expand Down Expand Up @@ -232,6 +248,9 @@ dependencies {
compile group: 'com.typesafe.akka', name: "akka-contrib_${scalaBinaryVersion}", version: akkaBinaryVersion
// compile group: 'org.iq80.leveldb', name: 'leveldb', version: '0.9'

compile group: 'com.typesafe.akka', name: "akka-http_${scalaBinaryVersion}", version: "10.1.8"
compile group: 'de.heikoseeberger', name: "akka-http-circe_${scalaBinaryVersion}", version: "1.25.2"

// TEST Akka //
testCompile group: 'com.typesafe.akka', name: "akka-testkit_${scalaBinaryVersion}", version: akkaBinaryVersion

Expand All @@ -249,6 +268,8 @@ dependencies {
compile group: 'org.apache.parquet', name: 'parquet-avro', version: parquet
compile (group: 'org.apache.hadoop', name: 'hadoop-client', version: '2.7.3') {
exclude group: 'org.slf4j', module: 'slf4j-log4j12'
// Exclude `ASM` because it is binary incompatible with the one which is gotten from `com.conveyal:kryo-tools`: `org.ow2.asm:asm:5.0.4`
exclude group: 'asm', module: 'asm'
}
}

Expand Down Expand Up @@ -292,14 +313,12 @@ build.dependsOn spec

task taggedTest(dependsOn: ['testClasses'], type: JavaExec) {
main = 'org.scalatest.tools.Runner'

args = ['-R', 'build/classes/scala/test', '-o', '-n'] << (project.findProperty('tags') ?: 'org.scalatest.Ignore')
classpath = sourceSets.test.runtimeClasspath
}

task specificTest(dependsOn: ['testClasses'], type: JavaExec) {
main = 'org.scalatest.tools.Runner'

args = ['-R', 'build/classes/scala/test', '-o', '-s'] << (project.findProperty('suite') ?: 'org.scalatest.Ignore')
classpath = sourceSets.test.runtimeClasspath
}
Expand All @@ -317,7 +336,6 @@ task periodicTest(dependsOn: ['testClasses'], type: JavaExec) {
'-Diterations=' + project.findProperty('iterations') : '')
jvmArgs = ['-javaagent:build/aspectjweaver-1.8.10.jar']
classpath = sourceSets.test.runtimeClasspath

doFirst() {
if (!project.file('build/aspectjweaver-1.8.10.jar').exists()) {
download {
Expand Down Expand Up @@ -549,7 +567,8 @@ def jfrWithMem = ["-XX:+UnlockCommercialFeatures", "-XX:+UnlockDiagnosticVMOptio

// UseParallelGC
applicationDefaultJvmArgs = ["-Xmx${myAvailableRam}g", "-Xms${myAvailableRam/2}g",
"-XX:+UseParallelGC", "-XX:+UseParallelOldGC", "-XX:MetaspaceSize=150M"] + logGC
"-XX:+UseParallelGC", "-XX:+UseParallelOldGC", "-XX:MetaspaceSize=150M", "-Djava.awt.headless=true",
"-Dlogback.configurationFile=logback_prod.xml"] + logGC

println(applicationDefaultJvmArgs)

Expand Down
6 changes: 0 additions & 6 deletions src/main/java/beam/agentsim/events/LoggerLevels.java

This file was deleted.

Original file line number Diff line number Diff line change
Expand Up @@ -87,7 +87,7 @@ private void setEventsFileFormats() {
}

Set<String> getKeysToWrite(Event event, Map<String, String> eventAttributes) {
return new HashSet<>(eventAttributes.keySet());
return eventAttributes.keySet();
}

/**
Expand Down
2 changes: 1 addition & 1 deletion src/main/java/beam/analysis/StatsFactory.java
Original file line number Diff line number Diff line change
Expand Up @@ -94,7 +94,7 @@ private BeamAnalysis createStats(StatsType statsType) {
case NumberOfVehicles:
return new NumberOfVehiclesAnalysis(beamServices);
case PersonCost:
return new PersonCostAnalysis();
return new PersonCostAnalysis(beamServices);
case AboveCapacityPtUsageDuration:
return new AboveCapacityPtUsageDurationAnalysis();
case TollRevenue:
Expand Down
17 changes: 13 additions & 4 deletions src/main/java/beam/analysis/physsim/PhyssimCalcLinkStats.java
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
package beam.analysis.physsim;

import beam.sim.BeamConfigChangesObservable;
import beam.sim.config.BeamConfig;
import beam.utils.BeamCalcLinkStats;
import beam.utils.VolumesAnalyzerFixed;
Expand All @@ -19,14 +20,14 @@
import org.matsim.core.utils.misc.Time;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import scala.Tuple2;
import java.awt.*;
import java.io.File;
import java.io.IOException;
import java.util.*;
import java.util.List;

public class PhyssimCalcLinkStats {
public class PhyssimCalcLinkStats implements Observer {

private Logger log = LoggerFactory.getLogger(PhyssimCalcLinkStats.class);

Expand Down Expand Up @@ -59,7 +60,7 @@ public class PhyssimCalcLinkStats {
private VolumesAnalyzer volumes;

public PhyssimCalcLinkStats(Network network, OutputDirectoryHierarchy controlerIO, BeamConfig beamConfig,
TravelTimeCalculatorConfigGroup ttcConfigGroup) {
TravelTimeCalculatorConfigGroup ttcConfigGroup, BeamConfigChangesObservable beamConfigChangesObservable) {
this.network = network;
this.controllerIO = controlerIO;
this.beamConfig = beamConfig;
Expand All @@ -73,6 +74,7 @@ public PhyssimCalcLinkStats(Network network, OutputDirectoryHierarchy controlerI
_noOfTimeBins = Math.floor(_noOfTimeBins);
noOfBins = _noOfTimeBins.intValue() + 1;
}
beamConfigChangesObservable.addObserver(this);

linkStats = new BeamCalcLinkStats(network, ttcConfigGroup);
}
Expand All @@ -99,7 +101,8 @@ private boolean isNotTestMode() {


private boolean writeLinkStats(int iterationNumber) {
return writeInIteration(iterationNumber, beamConfig.beam().physsim().linkStatsWriteInterval());
int interval = beamConfig.beam().physsim().linkStatsWriteInterval();
return writeInIteration(iterationNumber, interval);
}

private boolean writeInIteration(int iterationNumber, int interval) {
Expand Down Expand Up @@ -277,4 +280,10 @@ public void notifyIterationStarts(EventsManager eventsManager, TravelTimeCalcula
public void clean(){
this.linkStats.reset();
}

@Override
public void update(Observable observable, Object o) {
Tuple2 t = (Tuple2) o;
this.beamConfig = (BeamConfig) t._2;
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,128 @@
package beam.analysis.physsim;

import beam.analysis.plots.GraphUtils;
import beam.sim.config.BeamConfig;
import org.jfree.chart.ChartFactory;
import org.jfree.chart.JFreeChart;
import org.jfree.chart.plot.PlotOrientation;
import org.jfree.chart.plot.XYPlot;
import org.jfree.chart.renderer.xy.XYLineAndShapeRenderer;
import org.jfree.data.xy.XYSeries;
import org.jfree.data.xy.XYSeriesCollection;
import org.matsim.api.core.v01.network.Link;
import org.matsim.api.core.v01.network.Network;
import org.matsim.core.controler.OutputDirectoryHierarchy;
import org.matsim.core.utils.geometry.CoordUtils;
import org.matsim.core.utils.io.IOUtils;

import java.awt.geom.Ellipse2D;
import java.io.BufferedWriter;
import java.io.IOException;

public class PhyssimNetworkComparisonEuclideanVsLengthAttribute {
private BeamConfig beamConfig;
private Network network;
private OutputDirectoryHierarchy outputDirectoryHierarchy;

public PhyssimNetworkComparisonEuclideanVsLengthAttribute(Network network, OutputDirectoryHierarchy outputDirectoryHierarchy, BeamConfig beamConfig) {
this.network = network;
this.outputDirectoryHierarchy = outputDirectoryHierarchy;
this.beamConfig = beamConfig;
}

/**
* Iteration stop notification event listener
* @param iteration the count of the current iteration
*/
public void notifyIterationEnds(int iteration) {
if (beamConfig.beam().outputs().writeGraphs()) {
try {
writeComparisonEuclideanVsLengthAttributeCsv(iteration);
writeComparisonEuclideanVsLengthAttributePlot(iteration);
} catch (IOException e) {
e.printStackTrace();
}
}
}

/**
* Write CSV file with comparison of euclidean and length attribute of the network
* @param iterationNumber
* @throws IOException
*/
private void writeComparisonEuclideanVsLengthAttributeCsv(int iterationNumber) throws IOException {
String pathToCsv = outputDirectoryHierarchy.getIterationFilename(
iterationNumber,
"EuclideanVsLengthAttribute.csv"
);

BufferedWriter writerObservedVsSimulated = IOUtils.getBufferedWriter(pathToCsv);
writerObservedVsSimulated.write("linkId,euclidean,length\n");

for (Link link : network.getLinks().values()) {
writerObservedVsSimulated.write(
String.format("%s,%f,%f\n",
link.getId().toString(),
CoordUtils.calcEuclideanDistance(
link.getFromNode().getCoord(),
link.getToNode().getCoord()
), link.getLength())
);
}

writerObservedVsSimulated.close();
}

/**
* Make plot with comparison of euclidean and length attribute of the network
* @param iterationNumber
* @throws IOException
*/
private void writeComparisonEuclideanVsLengthAttributePlot(int iterationNumber) throws IOException {
String pathToPlot = outputDirectoryHierarchy.getIterationFilename(
iterationNumber,
"EuclideanVsLengthAttributePlot.png"
);

XYSeries series = new XYSeries("Euclidean vs Length attribute", false);

for (Link link : network.getLinks().values()) {
series.add(
CoordUtils.calcEuclideanDistance(
link.getFromNode().getCoord(),
link.getToNode().getCoord()
), link.getLength());
}

XYSeriesCollection dataset = new XYSeriesCollection();
dataset.addSeries(series);

JFreeChart chart = ChartFactory.createScatterPlot(
"Euclidean vs Length attribute",
"Euclidean",
"Length",
dataset,
PlotOrientation.VERTICAL,
true,
true,
false
);

XYPlot xyplot = chart.getXYPlot();
xyplot.setDomainCrosshairVisible(false);
xyplot.setRangeCrosshairVisible(false);

XYLineAndShapeRenderer renderer = new XYLineAndShapeRenderer();
renderer.setSeriesShape(0, new Ellipse2D.Double(0, 0, 5, 5));
renderer.setSeriesLinesVisible(0, false);

xyplot.setRenderer(0, renderer);

GraphUtils.saveJFreeChartAsPNG(
chart,
pathToPlot,
1000,
1000
);
}
}
Loading

0 comments on commit 7255d94

Please sign in to comment.