Skip to content

Commit

Permalink
feat(FgForrest#3): add Deepl automation integration
Browse files Browse the repository at this point in the history
  • Loading branch information
Stepan Kamenik committed Feb 28, 2024
1 parent 6654bb8 commit 9152e3a
Show file tree
Hide file tree
Showing 10 changed files with 188 additions and 39 deletions.
5 changes: 5 additions & 0 deletions pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -70,6 +70,11 @@
</properties>

<dependencies>
<dependency>
<groupId>com.deepl.api</groupId>
<artifactId>deepl-java</artifactId>
<version>1.4.0</version>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter</artifactId>
Expand Down
4 changes: 2 additions & 2 deletions src/main/java/one/edee/babylon/MainService.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,12 +29,12 @@ public MainService(Exporter exporter,
this.importProcessor = importProcessor;
}

public void startTranslation(Action action, TranslationConfiguration configuration, String spreadsheetId, boolean combineSheets) throws IOException, GeneralSecurityException, InterruptedException {
public void startTranslation(Action action, TranslationConfiguration configuration, String spreadsheetId, boolean combineSheets, String deeplApiKey) throws IOException, GeneralSecurityException, InterruptedException {
long stTime = System.currentTimeMillis();
switch (action) {
case EXPORT:
log.info("Babylon starting...");
exporter.walkPathsAndWriteSheets(configuration.getPath(), configuration.getMutations(), spreadsheetId, configuration.getSnapshotPath(), configuration.getLockedCellEditors(), combineSheets);
exporter.walkPathsAndWriteSheets(configuration.getPath(), configuration.getMutations(), spreadsheetId, configuration.getSnapshotPath(), configuration.getLockedCellEditors(), combineSheets, deeplApiKey);
break;
case IMPORT:
importProcessor.doImport(spreadsheetId);
Expand Down
21 changes: 16 additions & 5 deletions src/main/java/one/edee/babylon/SpringBootConsoleApplication.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,7 @@ public void run(String... args) {
try {
log.info("Loading config file: '" + arguments.getConfigFileName() + "'");
TranslationConfiguration configuration = configurationReader.readAndCheckConfiguration(arguments.getConfigFileName());
mainService.startTranslation(arguments.getAction(), configuration, arguments.getGoogleSheetId(), arguments.isCombineSheets());
mainService.startTranslation(arguments.getAction(), configuration, arguments.getGoogleSheetId(), arguments.isCombineSheets(), arguments.getDeeplApiKey());
} catch (Exception e) {
log.error("BABYLON ERROR: ", e);
System.exit(-1);
Expand All @@ -57,7 +57,7 @@ public void run(String... args) {
* @return
*/
public static Arguments parseArguments(String... args) {
if (!(args.length == 3 || args.length == 4)) {
if (!(args.length == 3 || args.length == 4 || args.length == 5)) {
log.error("Invalid input arguments, required: ");
printRequiredArguments();
System.exit(-1);
Expand All @@ -74,8 +74,12 @@ public static Arguments parseArguments(String... args) {
}
arguments.setConfigFileName(args[1]);
arguments.setGoogleSheetId(args[2]);
if (args.length == 4){
if (args.length > 3){
arguments.setCombineSheets(Boolean.parseBoolean(args[3]));
if (args.length > 4){
arguments.setDeeplApiKey(args[4]);

}
}
return arguments;
}
Expand All @@ -85,6 +89,7 @@ private static void printRequiredArguments() {
log.info("2 - path to translator-config.json file");
log.info("3 - ID of the google sheet (e.g. 1xhnBAOpy8-9KWhl8NP0ZIy6mhlgXKnKcLJwKcIeyjPc)");
log.info("4 - arg to specify combineSheets mode");
log.info("5 - arg to specify deepl api key");
}

/**
Expand All @@ -108,11 +113,17 @@ public static class Arguments {
* Id of the target google spreadsheet.
*/
private String googleSheetId;

/**
* Allows to write only to one sheet with name all.
* This mode is useful to correct duplicates etc.
* Allows writing only to one sheet with name all.
* This mode is useful to correct duplicates, etc.
*/
private boolean combineSheets = false;

/**
* Deepl api key.
*/
private String deeplApiKey;
}

}
83 changes: 70 additions & 13 deletions src/main/java/one/edee/babylon/export/Exporter.java
View file
Original file line number Diff line number Diff line change
@@ -1,14 +1,17 @@
package one.edee.babylon.export;

import com.deepl.api.TextResult;
import com.deepl.api.Translator;
import lombok.extern.apachecommons.CommonsLog;
import one.edee.babylon.db.SnapshotUtils;
import one.edee.babylon.export.dto.ExportResult;
import one.edee.babylon.export.dto.TranslationSheet;
import one.edee.babylon.sheets.SheetsException;
import one.edee.babylon.sheets.gsheets.model.ASheet;
import one.edee.babylon.snapshot.TranslationSnapshotWriteContract;
import one.edee.babylon.util.AntPathResourceLoader;
import one.edee.babylon.sheets.SheetsException;
import one.edee.babylon.util.PathUtils;
import lombok.extern.apachecommons.CommonsLog;
import org.springframework.util.StringUtils;

import java.io.File;
import java.io.IOException;
Expand Down Expand Up @@ -52,25 +55,28 @@ public void walkPathsAndWriteSheets(List<String> patternPaths,
List<String> translationLangs,
String spreadsheetId,
Path snapshotPath,
boolean combineSheets ) {
walkPathsAndWriteSheets(patternPaths, translationLangs, spreadsheetId, snapshotPath, Collections.emptyList(), combineSheets);
boolean combineSheets,
String deeplApiKey) {
walkPathsAndWriteSheets(patternPaths, translationLangs, spreadsheetId, snapshotPath, Collections.emptyList(), combineSheets, deeplApiKey);
}

/**
* Walks message file paths, gathering messages and translations, producing translation sheets in given GSheet spreadsheet.
*
* @param patternPaths paths of message files to export
* @param translationLangs languages to translate messages to
* @param spreadsheetId id of GSheets spreadsheet, must be empty
* @param snapshotPath path to the translation snapshot file
* @param patternPaths paths of message files to export
* @param translationLangs languages to translate messages to
* @param spreadsheetId id of GSheets spreadsheet, must be empty
* @param snapshotPath path to the translation snapshot file
* @param lockedCellEditors list of Google account emails, these account will have the permission to edit locked cells
* @param deeplApiKey
*/
public void walkPathsAndWriteSheets(List<String> patternPaths,
List<String> translationLangs,
String spreadsheetId,
Path snapshotPath,
List<String> lockedCellEditors,
boolean combineSheets) {
boolean combineSheets,
String deeplApiKey) {
warnDuplicatePaths(patternPaths);

List<ASheet> prevSheets = listAllSheets(spreadsheetId);
Expand Down Expand Up @@ -102,7 +108,56 @@ public void walkPathsAndWriteSheets(List<String> patternPaths,
original.add(new TranslationSheet(COMBINING_SHEET_NAME,combine));
}

uploadTranslations(result, spreadsheetId, lockedCellEditors);
Map<String, List<String>> changed = new HashMap<>();

if (deeplApiKey != null) {
try {
Translator translator = new Translator(deeplApiKey);
for (TranslationSheet sheet : result.getSheets()) {
log.info("Translating sheet " + sheet.getSheetName());

List<List<String>> rows = sheet.getRows();
List<String> header = rows.get(0);


for (int i = 1; i < rows.size(); i++) {
Map<Integer, String> toChange = new HashMap<>();

List<String> cells = rows.get(i);
String original = cells.get(1);
for (int l = 2; l < cells.size(); l++) {
if (StringUtils.isEmpty(cells.get(l))) {

String lang = header.get(l);

if (lang.equals("en")) {
lang = "en-GB";
}

if (StringUtils.hasText(original)) {
TextResult translatedText = translator.translateText(original, null, lang);
toChange.put(l, translatedText.getText());

changed
.computeIfAbsent(sheet.getSheetName(), key -> new LinkedList<>())
.add(i + "_" + l);
}
}
}

for (Entry<Integer, String> entry : toChange.entrySet()) {
cells.remove((int) entry.getKey());
cells.add(entry.getKey(), entry.getValue());
}

}
}
} catch (Exception e) {
log.error(e.getMessage(), e);
}
}

uploadTranslations(result, spreadsheetId, lockedCellEditors, changed);

updateSnapshotAndWriteToDisk(this.snapshot, result, snapshotPath);

Expand Down Expand Up @@ -162,13 +217,13 @@ private List<ASheet> listAllSheets(String spreadsheetId) {
}
}

private void uploadTranslations(ExportResult exportResult, String spreadsheetId, List<String> lockedCellEditors) {
private void uploadTranslations(ExportResult exportResult, String spreadsheetId, List<String> lockedCellEditors, Map<String, List<String>> changed) {
exportResult.getSheets().stream()
.filter(sheet -> !sheet.getDataRows().isEmpty())
.forEach(sheet -> {
try {
log.info("Writing " + sheet.getDataRows().size() + " rows into sheet '" + sheet.getSheetName() + "'.");
gsc.createSheet(spreadsheetId, sheet.getSheetName(), sheet.getRows(), lockedCellEditors);
gsc.createSheet(spreadsheetId, sheet.getSheetName(), sheet.getRows(), lockedCellEditors, changed);
} catch (SheetsException e) {
String errMsg = "Error when uploading data to spreadsheet '" + spreadsheetId + "'";
throw new RuntimeException(errMsg, e);
Expand Down Expand Up @@ -227,10 +282,12 @@ public interface SheetContract {
* @param sheetTitle name to use for the new sheet
* @param sheetRows rows with data cells to fill the sheet with
* @param lockedCellEditors list of email accounts that will be able to edit locked cells
* @param changed
* @throws SheetsException when unable to upload sheets
*/
void createSheet(String spreadsheetId, String sheetTitle, List<List<String>> sheetRows, List<String> lockedCellEditors) throws SheetsException;
void createSheet(String spreadsheetId, String sheetTitle, List<List<String>> sheetRows, List<String> lockedCellEditors, Map<String, List<String>> changed) throws SheetsException;

}


}
18 changes: 17 additions & 1 deletion src/main/java/one/edee/babylon/maven/BabylonExpImpBaseMojo.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,8 @@ public abstract class BabylonExpImpBaseMojo extends AbstractMojo {

public static final String CONFIG_FILE_PARAM = "config.file";
public static final String GOOGLE_SHEET_ID_PARAM = "google.sheet.id";
public static final String COMBINE_SHEET_PARAM = "combine.sheets";
public static final String DEEPL_API_KEY_PARAM = "deepl.api.key";

/**
* File name and relative path to the Json configuration file.
Expand All @@ -26,16 +28,30 @@ public abstract class BabylonExpImpBaseMojo extends AbstractMojo {
@Parameter(property = GOOGLE_SHEET_ID_PARAM, required = true)
private String googleSheetId;

/**
* Combine sheets param.
*/
@Parameter(property = COMBINE_SHEET_PARAM)
private Boolean combineSheets;

/**
* Deepl api key.
*/
@Parameter(property = DEEPL_API_KEY_PARAM)
private String deeplApiKey;

@Override
public void execute() {
SpringBootConsoleApplication.main(getArguments());
}

private String[] getArguments() {
String[] arg = new String[3];
String[] arg = new String[5];
arg[0] = getAction().name();
arg[1] = this.configFileName;
arg[2] = this.googleSheetId;
arg[3] = String.valueOf(this.combineSheets);
arg[4] = this.deeplApiKey;
return arg;
}

Expand Down
51 changes: 51 additions & 0 deletions src/main/java/one/edee/babylon/sheets/gsheets/GSheetApiRequestFactory.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,11 @@

import com.google.api.services.sheets.v4.model.*;

import java.util.LinkedList;
import java.util.List;
import java.util.Map;

import static java.util.Collections.singletonList;

/**
* Helps create requests for the Google Sheets API client library.
Expand Down Expand Up @@ -152,4 +156,51 @@ private DeleteSheetRequest deleteSheetRequest(Integer sheetId) {
.setSheetId(sheetId);
}

public List<Request> changeCellColor(Integer sheetId, String sheetTitle, Map<String, List<String>> changed) {
List<String> changes = changed.get(sheetTitle);
List<Request> reqs = new LinkedList<>();
if (changes != null && !changes.isEmpty()) {
for (String change : changes) {
String[] s = change.split("_");
int row = Integer.parseInt(s[0]);
int column = Integer.parseInt(s[1]);

reqs.add(
new Request()
.setUpdateCells(
new UpdateCellsRequest()
.setRange(
new GridRange()
.setSheetId(sheetId)
.setStartColumnIndex(column)
.setEndColumnIndex(column + 1)
.setStartRowIndex(row)
.setEndRowIndex(row + 1)
)
.setRows(
singletonList(
new RowData()
.setValues(
singletonList(
new CellData()
.setUserEnteredFormat(
new CellFormat()
.setBackgroundColor(
new Color()
.setRed(1f)
.setGreen(0.8f)
.setBlue(0.61f)
)
)
)
)
)
)
.setFields("userEnteredFormat.backgroundColor")
)
);
}
}
return reqs;
}
}
26 changes: 14 additions & 12 deletions src/main/java/one/edee/babylon/sheets/gsheets/LightGSheetService.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,10 +8,7 @@

import java.io.IOException;
import java.security.GeneralSecurityException;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.List;
import java.util.*;
import java.util.stream.Collectors;

/**
Expand Down Expand Up @@ -136,19 +133,24 @@ private List<? extends List<? extends Object>> convertNullsToEmptyString(List<?
* Updates style of an existing sheet.
* Sets wrapping strategy, resizes columns, protects firs two columns, hides first column.
*
* @param spreadsheetId id of spreadsheet to find sheet to update style
* @param sheetId id of sheet to update
* @param spreadsheetId id of spreadsheet to find sheet to update style
* @param sheetTitle
* @param sheetId id of sheet to update
* @param lockedCellEditors list of account emails to receive edit permissions on locked cells
* @param changed
* @throws GeneralSecurityException
* @throws IOException
*/
public void updateSheetStyle(String spreadsheetId, Integer sheetId, List<String> lockedCellEditors) throws GeneralSecurityException, IOException {
Request setWrappingStrategy = gSheetsRequestFactory.setWrapWrappingStrategyForAllCells(sheetId);
Request resizeColumns = gSheetsRequestFactory.resizeAllColumns(sheetId, COLUMN_WIDTH);
Request protectColumns = gSheetsRequestFactory.protectCellsInFirstTwoColumns(sheetId, lockedCellEditors);
Request hideColumn = gSheetsRequestFactory.hideFirstColumn(sheetId);
public void updateSheetStyle(String spreadsheetId, String sheetTitle, Integer sheetId, List<String> lockedCellEditors, Map<String, List<String>> changed) throws GeneralSecurityException, IOException {
List<Request> requests = new LinkedList<>();
requests.add(gSheetsRequestFactory.setWrapWrappingStrategyForAllCells(sheetId));
requests.add(gSheetsRequestFactory.resizeAllColumns(sheetId, COLUMN_WIDTH));
requests.add(gSheetsRequestFactory.protectCellsInFirstTwoColumns(sheetId, lockedCellEditors));
requests.add(gSheetsRequestFactory.hideFirstColumn(sheetId));
requests.addAll(gSheetsRequestFactory.changeCellColor(sheetId, sheetTitle, changed));

executeRequests(spreadsheetId, requests.toArray(new Request[0]));

executeRequests(spreadsheetId, setWrappingStrategy, resizeColumns, protectColumns, hideColumn);
}

public void deleteSheets(String spreadsheetId, Collection<Integer> sheetIds) throws GeneralSecurityException, IOException {
Expand Down
Loading

0 comments on commit 9152e3a

Please sign in to comment.