Merge pull request #34 from project-codeflare/singleton_bug

Author: raghukiran1224

codeflare/pipelines/Datamodel.py

@@ -837,7 +837,9 @@ def get_post_edges(self, node: Node):
         :return: Outgoing edges for the node
         """
         post_edges = []
-        post_nodes = self.__post_graph__[node]
+        post_nodes = []
+        if node in self.__post_graph__.keys():
+            post_nodes = self.__post_graph__[node]
         # Empty post
         if not post_nodes:
             post_edges.append(Edge(node, None))

codeflare/pipelines/Runtime.py

@@ -58,7 +58,8 @@ class ExecutionType(Enum):
     """
     FIT = 0,
     PREDICT = 1,
-    SCORE = 2
+    SCORE = 2,
+    TRANSFORM = 3
 
 
 @ray.remote
@@ -140,7 +141,10 @@ def execute_or_node_remote(node: dm.EstimatorNode, mode: ExecutionType, xy_ref:
             res_Xref = ray.put(estimator.transform(X))
             result = dm.XYRef(res_Xref, xy_ref.get_yref(), prev_node_ptr, prev_node_ptr, [xy_ref])
             return result
-
+    elif mode == ExecutionType.TRANSFORM:
+        res_Xref = ray.put(estimator.fit_transform(X))
+        result = dm.XYRef(res_Xref, xy_ref.get_yref(), prev_node_ptr, prev_node_ptr, [xy_ref])
+        return result
 
 def execute_or_node(node, pre_edges, edge_args, post_edges, mode: ExecutionType, is_outputNode):
     """

codeflare/pipelines/tests/test_singleton.py

@@ -0,0 +1,46 @@
+import pytest
+import ray
+import pandas as pd
+import numpy as np
+import sklearn.base as base
+from sklearn.preprocessing import MinMaxScaler
+import codeflare.pipelines.Datamodel as dm
+import codeflare.pipelines.Runtime as rt
+from codeflare.pipelines.Datamodel import Xy
+from codeflare.pipelines.Datamodel import XYRef
+from codeflare.pipelines.Runtime import ExecutionType
+
+def test_singleton():
+
+    ray.shutdown()
+    ray.init()
+
+    ## prepare the data
+    X = np.random.randint(0,100,size=(10000, 4))
+    y = np.random.randint(0,2,size=(10000, 1))
+
+    ## initialize codeflare pipeline by first creating the nodes
+    pipeline = dm.Pipeline()
+    node_a = dm.EstimatorNode('a', MinMaxScaler())
+    pipeline.add_node(node_a)
+
+    pipeline_input = dm.PipelineInput()
+    xy = dm.Xy(X,y)
+    pipeline_input.add_xy_arg(node_a, xy)
+
+    ## execute the codeflare pipeline
+    pipeline_output = rt.execute_pipeline(pipeline, ExecutionType.TRANSFORM, pipeline_input)
+
+    ## retrieve node e
+    node_a_output = pipeline_output.get_xyrefs(node_a)
+    Xout = ray.get(node_a_output[0].get_Xref())
+    yout = ray.get(node_a_output[0].get_yref())
+
+    assert Xout.shape[0] == 10000
+    assert yout.shape[0] == 10000
+
+    ray.shutdown()
+
+if __name__ == "__main__":
+    sys.exit(pytest.main(["-v", __file__]))
+