Add distributed simulator, change rmproxy to connect either to the sc…

…heduler or to the least loaded rt and correct some bugs

hadoop-assemblies/src/main/resources/assemblies/hadoop-kms-dist.xml

@@ -0,0 +1,52 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+  Licensed under the Apache License, Version 2.0 (the "License");
+  you may not use this file except in compliance with the License.
+  You may obtain a copy of the License at
+
+  http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+-->
+<assembly>
+  <id>hadoop-kms-dist</id>
+  <formats>
+    <format>dir</format>
+  </formats>
+  <includeBaseDirectory>false</includeBaseDirectory>
+  <fileSets>
+    <!-- Configuration files -->
+    <fileSet>
+      <directory>${basedir}/src/main/conf</directory>
+      <outputDirectory>/etc/hadoop</outputDirectory>
+      <includes>
+        <include>*</include>
+      </includes>
+    </fileSet>
+    <fileSet>
+      <directory>${basedir}/src/main/sbin</directory>
+      <outputDirectory>/sbin</outputDirectory>
+      <includes>
+        <include>*</include>
+      </includes>
+      <fileMode>0755</fileMode>
+    </fileSet>
+    <fileSet>
+      <directory>${basedir}/src/main/libexec</directory>
+      <outputDirectory>/libexec</outputDirectory>
+      <includes>
+        <include>*</include>
+      </includes>
+      <fileMode>0755</fileMode>
+    </fileSet>
+    <!-- Documentation -->
+    <fileSet>
+      <directory>${project.build.directory}/site</directory>
+      <outputDirectory>/share/doc/hadoop/kms</outputDirectory>
+    </fileSet>
+  </fileSets>
+</assembly>

hadoop-common-project/hadoop-common/pom.xml

@@ -285,6 +285,12 @@
       <artifactId>bcprov-jdk16</artifactId>
       <scope>test</scope>
     </dependency>
+    <dependency>
+      <groupId>io.hops.metadata</groupId>
+      <artifactId>hops-metadata-dal</artifactId>
+      <version>1.0-SNAPSHOT</version>
+      <type>jar</type>
+    </dependency>
   </dependencies>
 
   <build>

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/service/AbstractService.java

@@ -399,7 +399,7 @@ public String getName() {
   }
 
   @Override
-  public synchronized Configuration getConfig() {
+  public Configuration getConfig() {
     return config;
   }
 

hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMCommunicator.java

@@ -310,7 +310,7 @@ protected ApplicationMasterProtocol createSchedulerProxy() {
     final Configuration conf = getConfig();
 
     try {
-      return ClientRMProxy.createRMProxy(conf, ApplicationMasterProtocol.class);
+      return ClientRMProxy.createRMProxy(conf, ApplicationMasterProtocol.class,true);
     } catch (IOException e) {
       throw new YarnRuntimeException(e);
     }

hadoop-tools/distributedLoadSimulator/.gitignore

@@ -0,0 +1 @@
+distributedsimulator_resources

hadoop-tools/distributedLoadSimulator/pom.xml

@@ -0,0 +1,41 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <modelVersion>4.0.0</modelVersion>
+    <parent>
+        <groupId>io.hops</groupId>
+        <artifactId>hadoop-tools</artifactId>
+        <version>2.4.0</version>
+    </parent>
+    <artifactId>distributedLoadSimulator</artifactId>
+    <packaging>jar</packaging>
+    <dependencies>
+        <dependency>
+            <groupId>${project.groupId}</groupId>
+            <artifactId>hadoop-common</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>${project.groupId}</groupId>
+            <artifactId>hadoop-rumen</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>${project.groupId}</groupId>
+            <artifactId>hadoop-yarn-api</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>${project.groupId}</groupId>
+            <artifactId>hadoop-yarn-server-resourcemanager</artifactId>
+            <type>jar</type>
+        </dependency>
+        <dependency>
+            <groupId>${project.groupId}</groupId>
+            <artifactId>hadoop-yarn-client</artifactId>
+            <type>jar</type>
+        </dependency>
+        <dependency>
+            <groupId>com.codahale.metrics</groupId>
+            <artifactId>metrics-core</artifactId>
+            <version>3.0.1</version>
+            <type>jar</type>
+        </dependency>
+    </dependencies>
+</project>

hadoop-tools/distributedLoadSimulator/src/main/java/org/apache/hadoop/distributedloadsimulator/sls/AMNMCommonObject.java

@@ -0,0 +1,74 @@
+/*
+ * Copyright 2015 Apache Software Foundation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.distributedloadsimulator.sls;
+
+import java.rmi.Remote;
+import java.rmi.RemoteException;
+import java.util.List;
+
+/**
+ *
+ * @author sri
+ */
+public interface AMNMCommonObject extends Remote {
+
+ // all the unmanged application master will call this to update node manager's containers 
+  // internal data strcutres
+  void cleanupContainer(String containerId, String nodeId) throws
+          RemoteException;
+
+  boolean isNodeExist(String nodeId) throws RemoteException;
+
+  void addNewContainer(String containerId, String nodeId, String httpAddress,
+          int memory, int vcores, int priority, long lifeTimeMS) throws
+          RemoteException;
+
+  void decreseApplicationCount(String applicationId, boolean failed) throws
+          RemoteException;
+
+  int finishedApplicationsCount() throws RemoteException;
+
+  void registerApplicationTimeStamp() throws RemoteException;
+
+  boolean isNMRegisterationDone() throws RemoteException;
+
+  void simulationFinished() throws RemoteException;
+
+  int[] getHandledHeartBeats() throws RemoteException;
+
+  int getNumberNodeManager() throws RemoteException;
+
+  public void addApplicationMasterWaitTime(long applicationMasterWaitTime)
+          throws RemoteException;
+
+  public void addContainerAllocationWaitTime(long containerAllocationWaitTime)
+          throws RemoteException;
+
+  public void addContainerStartWaitTime(long containerStartWaitTime) throws
+          RemoteException;
+
+  public Long getApplicationMasterWaitTime() throws RemoteException;
+
+  public Long getContainerAllocationWaitTime() throws RemoteException;
+
+  public Long getContainerStartWaitTime() throws RemoteException;
+
+  public int getNBContainers() throws RemoteException;
+
+  public int getNBApplicationMasterWaitTime() throws RemoteException;
+
+  public void kill() throws RemoteException;
+}