Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Changes for: Write recommendations to BQ table as a record #16

Open
wants to merge 3 commits into
base: anti-patterns
Choose a base branch
from
Open

Changes for: Write recommendations to BQ table as a record #16

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
34defab
Changes for: Write recommendations to BQ table as a record
darshan-ggl May 8, 2023
7d4687a
Changes for: Write recommendations to BQ table as a record
darshan-ggl May 8, 2023
25b61af
Updated the antipattern_output_table DDL
darshan-ggl May 8, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 3 additions & 1 deletion tools/zetasql-helper/bigquery-antipattern-recognition/README.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -56,7 +56,9 @@ Read from information schema and write to output table:
CREATE TABLE dataset.antipattern_output_table (
job_id STRING,
query STRING,
recommendation STRING,
recommendation ARRAY<STRUCT<
name STRING,
string STRING>>,
slot_hours FLOAT64,
process_timestamp TIMESTAMP
);
Expand Down
16 changes: 8 additions & 8 deletions ...recognition/src/main/java/com/google/zetasql/toolkit/antipattern/cmd/OutputGenerator.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,21 +28,21 @@

public class OutputGenerator {

public static void writeOutput(BQAntiPatternCMDParser cmdParser, List<String[]> outputData)
public static void writeOutput(BQAntiPatternCMDParser cmdParser, List<Object[]> outputData)
throws IOException {
if (cmdParser.hasOutputFileOptionName()) {
writeOutputToCSV(outputData, cmdParser);
} else if (cmdParser.isReadingFromInfoSchema() && cmdParser.hasOutputTable()) {
writeOutputToBQTable(outputData, cmdParser);
} else {
for (String[] row : outputData) {
for (Object[] row : outputData) {
System.out.println("----------------------------------------");
System.out.println(String.join("\n", row));
System.out.println(String.join("\n", (String[]) row));
}
}
}

private static void writeOutputToCSV(List<String[]> outputData, BQAntiPatternCMDParser cmdParser)
private static void writeOutputToCSV(List<Object[]> outputData, BQAntiPatternCMDParser cmdParser)
throws IOException {

FileWriter csvWriter;
Expand All @@ -54,17 +54,17 @@ private static void writeOutputToCSV(List<String[]> outputData, BQAntiPatternCMD
csvWriter.write(String.join(",", new String[] {"id", "query\n"}));
}

for (String[] row : outputData) {
csvWriter.write(String.join(",", row));
for (Object[] row : outputData) {
csvWriter.write(String.join(",", (String[]) row));
csvWriter.write("\n");
}
csvWriter.close();
}

private static void writeOutputToBQTable(
List<String[]> outputData, BQAntiPatternCMDParser cmdParser) {
List<Object[]> outputData, BQAntiPatternCMDParser cmdParser) {
DateTime date = new DateTime(new Date());
for (String[] row : outputData) {
for (Object[] row : outputData) {
Map<String, Object> rowContent = new HashMap<>();
rowContent.put("job_id", row[0]);
rowContent.put("query", row[1]);
Expand Down
56 changes: 40 additions & 16 deletions ...pattern-recognition/src/main/java/com/google/zetasql/toolkit/antipattern/parser/Main.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,8 @@
import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.HashMap;
import java.util.stream.Collectors;
import org.apache.commons.cli.ParseException;
import org.slf4j.Logger;
Expand All @@ -48,12 +50,12 @@ public static void main(String[] args) throws ParseException, IOException {
while (inputQueriesIterator.hasNext()) {
inputQuery = inputQueriesIterator.next();
String query = inputQuery.getQuery();
List<String[]> outputData = new ArrayList<>();
List<Object[]> outputData = new ArrayList<>();

try {
ASTStatement parsedQuery = Parser.parseStatement(query, languageOptions);
String rec = getRecommendations(parsedQuery);
if (rec.length() > 0) {
List<Map<String, String>> rec = getRecommendations(parsedQuery);
if (rec.size() > 0) {
addRecToOutput(cmdParser, outputData, inputQuery, rec);
OutputGenerator.writeOutput(cmdParser, outputData);
countAntiPatterns += 1;
Expand All @@ -75,29 +77,51 @@ public static void main(String[] args) throws ParseException, IOException {

private static void addRecToOutput(
BQAntiPatternCMDParser cmdParser,
List<String[]> outputData,
List<Object[]> outputData,
InputQuery inputQuery,
String rec) {
List<Map<String, String>> rec) {
if (cmdParser.isReadingFromInfoSchema()) {
outputData.add(
new String[] {
new Object[] {
inputQuery.getQueryId(),
inputQuery.getQuery(),
Float.toString(inputQuery.getSlotHours()),
"\"" + rec + "\"",
rec,
});
} else {
outputData.add(new String[] {inputQuery.getQueryId(), "\"" + rec + "\""});
String output = rec.stream().map(m -> m.get("name") + ": \"" + m.getOrDefault("description", "") + "\"\n").collect(Collectors.joining());
outputData.add(new String[] {inputQuery.getQueryId(), output});
}
}

private static String getRecommendations(ASTStatement parsedQuery) {
ArrayList<String> recommendation = new ArrayList<>();
recommendation.add(new IdentifySimpleSelectStar().run(parsedQuery));
recommendation.add(new IdentifyInSubqueryWithoutAgg().run(parsedQuery));
recommendation.add(new IdentifyCrossJoin().run(parsedQuery));
recommendation.add(new IdentifyCTEsEvalMultipleTimes().run(parsedQuery));
recommendation.add(new IdentifyOrderByWithoutLimit().run(parsedQuery));
return recommendation.stream().filter(x -> x.length() > 0).collect(Collectors.joining("\n"));
private static List<Map<String, String>> getRecommendations(ASTStatement parsedQuery) {
List<Map<String, String>> recommendations = new ArrayList<>();
recommendations.add(new HashMap<>() {{
put("name", "SelectStar");
put("description", new IdentifySimpleSelectStar().run(parsedQuery));
}});
recommendations.add(new HashMap<>() {{
put("name", "SelectStar");
put("description", new IdentifySimpleSelectStar().run(parsedQuery));
}});
recommendations.add(new HashMap<>() {{
put("name", "SubqueryWithoutAgg");
put("description", new IdentifyInSubqueryWithoutAgg().run(parsedQuery));
}});
recommendations.add(new HashMap<>() {{
put("name", "CrossJoin");
put("description", new IdentifyCrossJoin().run(parsedQuery));
}});
recommendations.add(new HashMap<>() {{
put("name", "CTEsEvalMultipleTimes");
put("description", new IdentifyCTEsEvalMultipleTimes().run(parsedQuery));
}});
recommendations.add(new HashMap<>() {{
put("name", "OrderByWithoutLimit");
put("description", new IdentifyOrderByWithoutLimit().run(parsedQuery));
}});

recommendations.removeIf(m -> m.get("description").isEmpty());
return recommendations;
}
}