Skip to content

Commit e61c429

Browse files
rbparkrbpark
committed
Merge branch 'master' of gitli.corp.linkedin.com:azkaban/azkaban2
2 parents f5a6295 + ce30041 commit e61c429

File tree

11 files changed

+261
-31
lines changed

11 files changed

+261
-31
lines changed

‎src/java/azkaban/executor/ExecutorManager.java

Lines changed: 53 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -56,8 +56,12 @@
5656
import org.apache.http.impl.client.BasicResponseHandler;
5757
import org.apache.http.impl.client.DefaultHttpClient;
5858
import org.apache.log4j.Logger;
59+
import org.joda.time.DateTime;
60+
import org.joda.time.DateTimeZone;
61+
import org.joda.time.format.DateTimeFormat;
5962

6063
import azkaban.executor.ExecutableFlow.Status;
64+
import azkaban.executor.ExecutorManager.ExecutionReference;
6165
import azkaban.flow.Flow;
6266
import azkaban.utils.ExecutableFlowLoader;
6367
import azkaban.utils.JSONUtils;
@@ -291,25 +295,37 @@ private boolean between(ExecutionReference ref, long startTime, long endTime) {
291295
return endTime > refStart && startTime <= refEnd;
292296
}
293297

294-
public List<ExecutionReference> getFlowHistory(String regexPattern, int numResults, int skip) {
298+
public List<ExecutionReference> getFlowHistory(String projRe, String flowRe, String userRe, long startTime, long endTime, int numResults, int skip, Boolean dofilter) {
295299
ArrayList<ExecutionReference> searchFlows = new ArrayList<ExecutionReference>();
296300

297-
Pattern pattern;
298-
try {
299-
pattern = Pattern.compile(regexPattern, Pattern.CASE_INSENSITIVE);
300-
} catch (PatternSyntaxException e) {
301-
logger.error("Bad regex pattern " + regexPattern);
302-
return searchFlows;
301+
Pattern projPattern = null;
302+
Pattern flowPattern = null;
303+
Pattern userPattern = null;
304+
305+
if(dofilter == true) {
306+
try {
307+
projPattern = Pattern.compile(projRe, Pattern.CASE_INSENSITIVE);
308+
flowPattern = Pattern.compile(flowRe, Pattern.CASE_INSENSITIVE);
309+
userPattern = Pattern.compile(userRe, Pattern.CASE_INSENSITIVE);
310+
} catch (PatternSyntaxException e) {
311+
logger.error("Bad regex pattern " + projRe + " " + flowRe + " " + userRe);
312+
return searchFlows;
313+
}
303314
}
304315

305316
for (ExecutionReference ref: runningReference.values()) {
306317
if (skip > 0) {
307318
skip--;
308319
}
309320
else {
310-
if(pattern.matcher(ref.getFlowId()).find() ) {
311-
searchFlows.add(ref);
321+
if(dofilter == true) {
322+
if(flowPattern.matcher(ref.getFlowId()).find() && projPattern.matcher(ref.getProjectId()).find() && userPattern.matcher(ref.getUserId()).find()
323+
&& between(ref, startTime, endTime) ) {
324+
searchFlows.add(ref);
325+
}
312326
}
327+
else
328+
searchFlows.add(ref);
313329
if (searchFlows.size() == numResults) {
314330
Collections.sort(searchFlows);
315331
return searchFlows;
@@ -322,7 +338,25 @@ public List<ExecutionReference> getFlowHistory(String regexPattern, int numResul
322338
return searchFlows;
323339
}
324340

325-
File[] archivePartitionsDir = archivePath.listFiles();
341+
File[] archivePartitionsDir;
342+
if(dofilter == true) {
343+
final long startThreshold = startTime - 100000;
344+
final long endThreshold = endTime + 100000;
345+
346+
archivePartitionsDir = archivePath.listFiles( new FileFilter() {
347+
@Override
348+
public boolean accept(File pathname) {
349+
String name = pathname.getName();
350+
long val = Long.valueOf(name);
351+
return val >= startThreshold && val <= endThreshold;
352+
}
353+
}
354+
);
355+
}
356+
else {
357+
archivePartitionsDir = archivePath.listFiles();
358+
}
359+
326360
Arrays.sort(archivePartitionsDir, new Comparator<File>() {
327361
@Override
328362
public int compare(File arg0, File arg1) {
@@ -347,8 +381,14 @@ public int compare(File arg0, File arg1) {
347381
if (ref == null) {
348382
continue;
349383
}
350-
351-
if(pattern.matcher(ref.getFlowId()).find() ) {
384+
385+
if(dofilter == true) {
386+
if(flowPattern.matcher(ref.getFlowId()).find() && projPattern.matcher(ref.getProjectId()).find() && userPattern.matcher(ref.getUserId()).find()
387+
&& between(ref, startTime, endTime)) {
388+
searchFlows.add(ref);
389+
}
390+
}
391+
else {
352392
searchFlows.add(ref);
353393
}
354394
} catch (IOException e) {
@@ -1374,4 +1414,5 @@ public void setLastCheckedTime(long lastCheckedTime) {
13741414
}
13751415
}
13761416

1417+
13771418
}

‎src/java/azkaban/project/FileProjectManager.java

Lines changed: 20 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -219,7 +219,7 @@ public List<Project> getUserProjects(User user) {
219219

220220
@Override
221221
public List<Project> getUserProjectsByRe(User user, final String rePattern) {
222-
ArrayList<Project> array = new ArrayList<Project>();
222+
List<Project> array = new ArrayList<Project>();
223223
Pattern pattern;
224224
try {
225225
pattern = Pattern.compile(rePattern, Pattern.CASE_INSENSITIVE);
@@ -246,6 +246,23 @@ public List<Project> getProjects() {
246246
return new ArrayList<Project>(projects.values());
247247
}
248248

249+
@Override
250+
public List<Project> getProjectsByRe(String rePattern) {
251+
List<Project> allProjects = new ArrayList<Project>();
252+
Pattern pattern;
253+
try {
254+
pattern = Pattern.compile(rePattern, Pattern.CASE_INSENSITIVE);
255+
} catch (PatternSyntaxException e) {
256+
logger.error("Bad regex pattern " + rePattern);
257+
return allProjects;
258+
}
259+
for(Project project : getProjects()) {
260+
if(pattern.matcher(project.getName()).find()) {
261+
allProjects.add(project);
262+
}
263+
}
264+
return allProjects;
265+
}
249266

250267
@Override
251268
public Project getProject(String name) {
@@ -662,4 +679,6 @@ private String projectLogFileName(String projectName) {
662679
return "_project." + projectName + ".log";
663680
}
664681

682+
683+
665684
}

‎src/java/azkaban/project/ProjectManager.java

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,8 @@ public interface ProjectManager {
1818
public List<Project> getUserProjectsByRe(User user, String searchTerm);
1919

2020
public List<Project> getProjects();
21+
22+
public List<Project> getProjectsByRe(String searchTerm);
2123

2224
public void commitProject(String name) throws ProjectManagerException;
2325

@@ -39,5 +41,7 @@ public interface ProjectManager {
3941

4042
public void getProjectLogs(String projectId, long tailBytes, long skipBytes, Writer writer) throws IOException;
4143

44+
45+
4246

4347
}

‎src/java/azkaban/webapp/servlet/HistoryServlet.java

Lines changed: 21 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -2,8 +2,7 @@
22

33
import java.io.IOException;
44
import java.util.ArrayList;
5-
import java.util.Collections;
6-
import java.util.Comparator;
5+
76
import java.util.HashMap;
87
import java.util.List;
98

@@ -12,9 +11,12 @@
1211
import javax.servlet.http.HttpServletRequest;
1312
import javax.servlet.http.HttpServletResponse;
1413

15-
import azkaban.executor.ExecutableFlow;
14+
import org.joda.time.DateTime;
15+
16+
import org.joda.time.format.DateTimeFormat;
17+
18+
1619
import azkaban.executor.ExecutorManager;
17-
import azkaban.executor.ExecutorManagerException;
1820
import azkaban.executor.ExecutorManager.ExecutionReference;
1921
import azkaban.utils.JSONUtils;
2022
import azkaban.webapp.session.Session;
@@ -63,7 +65,7 @@ protected void handlePost(HttpServletRequest req, HttpServletResponse resp, Sess
6365
pageNum = 1;
6466
}
6567

66-
List<ExecutionReference> history = executorManager.getFlowHistory(searchTerm, pageSize, (pageNum - 1)*pageSize);
68+
List<ExecutionReference> history = executorManager.getFlowHistory(".*", searchTerm, ".*", 0, DateTime.now().getMillis(), pageSize, (pageNum - 1)*pageSize, true);
6769
page.add("flowHistory", history);
6870
page.add("size", pageSize);
6971
page.add("page", pageNum);
@@ -113,7 +115,7 @@ private void handleAJAXAction(HttpServletRequest req, HttpServletResponse resp,
113115
this.writeJSON(resp, ret);
114116
}
115117
}
116-
118+
117119
private void fetchHistoryData(HttpServletRequest req, HttpServletResponse resp, HashMap<String, Object> ret) throws ServletException {
118120
long start = getLongParam(req, "start");
119121
long end = getLongParam(req, "end");
@@ -137,16 +139,26 @@ private void fetchHistoryData(HttpServletRequest req, HttpServletResponse resp,
137139
ret.put("data", refList);
138140
}
139141

140-
private void handleHistoryPage(HttpServletRequest req, HttpServletResponse resp, Session session) {
142+
private void handleHistoryPage(HttpServletRequest req, HttpServletResponse resp, Session session) throws ServletException {
141143
Page page = newPage(req, resp, session, "azkaban/webapp/servlet/velocity/historypage.vm");
142144
int pageNum = getIntParam(req, "page", 1);
143145
int pageSize = getIntParam(req, "size", 16);
144146

145147
if (pageNum < 0) {
146148
pageNum = 1;
147149
}
148-
149-
List<ExecutionReference> history = executorManager.getFlowHistory(".*", pageSize, (pageNum - 1)*pageSize);
150+
List<ExecutionReference> history = null;
151+
if(hasParam(req, "advfilter")) {
152+
String projRe = getParam(req, "projre").equals("") ? ".*" : getParam(req, "projre");
153+
String flowRe = getParam(req, "flowre").equals("") ? ".*" : getParam(req, "flowre");
154+
String userRe = getParam(req, "userre").equals("") ? ".*" : getParam(req, "userre");
155+
long beginTime = getParam(req, "begin").equals("") ? 0 : DateTimeFormat.forPattern("MM/dd/yyyy").parseDateTime(getParam(req, "begin")).getMillis();
156+
long endTime = getParam(req, "end").equals("") ? DateTime.now().getMillis() : DateTimeFormat.forPattern("MM/dd/yyyy").parseDateTime(getParam(req, "end")).getMillis();
157+
history = executorManager.getFlowHistory(projRe, flowRe, userRe, beginTime, endTime, pageSize, (pageNum - 1)*pageSize, true);
158+
}
159+
else {
160+
history = executorManager.getFlowHistory("", "", "", 0, 0, pageSize, (pageNum - 1)*pageSize, false);
161+
}
150162
page.add("flowHistory", history);
151163
page.add("size", pageSize);
152164
page.add("page", pageNum);

‎src/java/azkaban/webapp/servlet/IndexServlet.java

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -47,7 +47,7 @@ protected void handleGet(HttpServletRequest req, HttpServletResponse resp, Sessi
4747
Page page = newPage(req, resp, session, "azkaban/webapp/servlet/velocity/index.vm");
4848
if (hasParam(req, "all")) {
4949
List<Project> projects = manager.getProjects();
50-
page.add("allProjects", "");
50+
page.add("allProjects", "true");
5151
page.add("projects", projects);
5252
}
5353
else {
@@ -72,9 +72,10 @@ protected void handlePost(HttpServletRequest req, HttpServletResponse resp, Sess
7272
Page page = newPage(req, resp, session, "azkaban/webapp/servlet/velocity/index.vm");
7373
if (hasParam(req, "all")) {
7474
//do nothing special if one asks for 'ALL' projects
75-
List<Project> projects = manager.getProjects();
75+
List<Project> projects = manager.getProjectsByRe(searchTerm);
7676
page.add("allProjects", "");
7777
page.add("projects", projects);
78+
page.add("search_term", searchTerm);
7879
}
7980
else {
8081
List<Project> projects = manager.getUserProjectsByRe(user, searchTerm);

‎src/java/azkaban/webapp/servlet/velocity/hdfsbrowserpage.vm

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -58,6 +58,7 @@
5858
<th>Permission</th>
5959
<th>Owner/Group</th>
6060
<th>Size</th>
61+
<th>Block Size</th>
6162
<th>Reps</th>
6263
<th>Modified Date</th>
6364
</tr>
@@ -78,6 +79,13 @@
7879
$utils.displayBytes(${status.len})
7980
#end
8081
</td>
82+
<td>
83+
#if($status.isDir())
84+
&ndash;
85+
#else
86+
$utils.displayBytes(${status.getBlockSize()})
87+
#end
88+
</td>
8189
<td>
8290
#if($status.isDir())
8391
&ndash;

‎src/java/azkaban/webapp/servlet/velocity/historypage.vm

Lines changed: 46 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -2,13 +2,15 @@
22
<html>
33
<head>
44
#parse( "azkaban/webapp/servlet/velocity/style.vm" )
5-
<script type="text/javascript" src="${context}/js/jquery/jquery.js"></script>
5+
<script type="text/javascript" src="${context}/js/jquery/jquery.js"></script>
6+
<script type="text/javascript" src="${context}/js/jqueryui/jquery-ui.custom.min.js"></script>
7+
<script type="text/javascript" src="${context}/js/jqueryui/jquery.ui.datepicker.min.js"></script>
68
<script type="text/javascript" src="${context}/js/namespace.js"></script>
79
<script type="text/javascript" src="${context}/js/underscore-1.2.1-min.js"></script>
810
<script type="text/javascript" src="${context}/js/backbone-0.5.3-min.js"></script>
911
<script type="text/javascript" src="${context}/js/jquery.simplemodal.js"></script>
10-
<script type="text/javascript" src="${context}/js/azkaban.nav.js"></script>
11-
<script type="text/javascript" src="${context}/js/azkaban.main.view.js"></script>
12+
<script type="text/javascript" src="${context}/js/azkaban.nav.js"></script>
13+
<script type="text/javascript" src="${context}/js/azkaban.history.view.js"></script>
1214
<script type="text/javascript">
1315
var contextURL = "${context}";
1416
var currentTime = ${currentTime};
@@ -26,14 +28,18 @@
2628
<div id="all-jobs-content">
2729
<div class="section-hd">
2830
<h2>History</h2>
31+
<a id="adv-filter-btn" class="btn1 " href="#">Advanced Filter</a>
2932
<form id="search-form" method="post">
3033
<input type="hidden" name="action" value="search">
3134
<input type="submit" value="Quick Search" class="search-btn">
32-
<input id="searchtextbox" type="text" placeholder="by flow name containing ..." value=#if($search_term) ${search_term} #else "" #end class="search-input" name="searchterm">
35+
<input id="searchtextbox" type="text" placeholder="flow name containing ..." value=#if($search_term) ${search_term} #else "" #end class="search-input" name="searchterm">
3336
</form>
37+
3438
</div>
3539
</div>
3640

41+
42+
3743
<div class="executionInfo">
3844
<table id="executingJobs">
3945
<thead>
@@ -92,5 +98,41 @@
9298
</div>
9399
</div>
94100
</div>
101+
102+
<!-- modal content -->
103+
<div id="adv-filter" class="modal">
104+
<h3>Advanced Filter</h3>
105+
<div id="errorMsg" class="box-error-message">$errorMsg</div>
106+
107+
<div class="message">
108+
<fieldset>
109+
<dl>
110+
111+
<dt><label for="path">Project Name</label></dt>
112+
<dd><input id="projRe" type="text" placeholder="project name containing ..." value = "" class="filter-input" name="projre"/></dd>
113+
<dt><label for="path">Flow Name Name</label></dt>
114+
<dd><input id="flowRe" type="text" placeholder="flow name containing ..." value = "" class="filter-input" name="flowre"/></dd>
115+
<dt><label for="path">User Name</label></dt>
116+
<dd><input id="userRe" type="text" placeholder="user name containing ..." value = "" class="filter-input" name="userre"/></dd>
117+
<dt>Date between</dt>
118+
<dd><input type="text" id="daterangebegin" value=""/></dd>
119+
<dt>and</dt>
120+
<dd><input type="text" id="daterangeend" value=""/></dd>
121+
</dl>
122+
</fieldset>
123+
</div>
124+
<div class="actions">
125+
<a class="yes btn2" id="filter-btn" href="#">Filter</a>
126+
<a class="no simplemodal-close btn3" href="#">Cancel</a>
127+
</div>
128+
<div id="invalid-session" class="modal">
129+
<h3>Invalid Session</h3>
130+
<p>Session has expired. Please re-login.</p>
131+
<div class="actions">
132+
<a class="yes btn2" id="login-btn" href="#">Re-login</a>
133+
</div>
134+
</div>
135+
</div>
136+
95137
</body>
96138
</html>

‎src/java/azkaban/webapp/servlet/velocity/index.vm

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -44,10 +44,10 @@
4444
<form id="search-form" method="post">
4545
<input type="hidden" name="action" value="search">
4646
<input type="submit" value="Quick Search" class="search-btn">
47-
<input id="searchtextbox" type="text" placeholder="by project name containing ..." value=#if($search_term) ${search_term} #else "" #end class="search-input" name="searchterm">
47+
<input id="searchtextbox" type="text" placeholder="project name containing ..." value=#if($search_term) ${search_term} #else "" #end class="search-input" name="searchterm">
4848
</form>
4949

50-
<a id="create-project-btn" class="btn1 createproject" href="#">Create Project</a>
50+
<a id="create-project-btn" class="btn1 " href="#">Create Project</a>
5151
</div><!-- end .section-hd -->
5252
</div>
5353
<table id="all-jobs" class="all-jobs job-table">

‎src/java/azkaban/webapp/servlet/velocity/scheduledflowpage.vm

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,6 @@
99
<script type="text/javascript" src="${context}/js/backbone-0.5.3-min.js"></script>
1010
<script type="text/javascript" src="${context}/js/jquery.simplemodal.js"></script>
1111
<script type="text/javascript" src="${context}/js/azkaban.nav.js"></script>
12-
<script type="text/javascript" src="${context}/js/azkaban.main.view.js"></script>
1312
<script type="text/javascript" src="${context}/js/azkaban.scheduled.view.js"></script>
1413
<script type="text/javascript">
1514
var contextURL = "${context}";

0 commit comments

Comments
 (0)