Skip to content

Commit

Permalink
Add decryption to dataset iterator
Browse files Browse the repository at this point in the history
  • Loading branch information
Marco Libanori committed Jan 16, 2024
1 parent 82e20c3 commit d2a70fc
Show file tree
Hide file tree
Showing 8 changed files with 229 additions and 203 deletions.
129 changes: 66 additions & 63 deletions knowageutils/src/main/java/it/eng/spagobi/tools/dataset/bo/AbstractJDBCDataset.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,8 @@

import javax.naming.NamingException;

import org.apache.log4j.Logger;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.json.JSONArray;
import org.json.JSONException;
import org.json.JSONObject;
Expand Down Expand Up @@ -66,33 +67,30 @@

public abstract class AbstractJDBCDataset extends ConfigurableDataSet {

public static String DS_TYPE = "SbiQueryDataSet";
private static final Logger LOGGER = LogManager.getLogger(AbstractJDBCDataset.class);
private static final String DS_TYPE = "SbiQueryDataSet";

protected SelectQuery selectQuery;

private static final String QUERY = "query";
private static final String QUERY_SCRIPT = "queryScript";
private static final String QUERY_SCRIPT_LANGUAGE = "queryScriptLanguage";
private static final String DATA_SOURCE = "dataSource";

private static transient Logger logger = Logger.getLogger(AbstractJDBCDataset.class);

/**
* Instantiates a new empty JDBC data set.
*/
public AbstractJDBCDataset() {
super();
protected AbstractJDBCDataset() {
setDataProxy(new JDBCDataProxy());
setDataReader(new JDBCStandardDataReader());
addBehaviour(new QuerableBehaviour(this));
}

// cannibalization :D
public AbstractJDBCDataset(JDBCDataSet jdbcDataset) {
protected AbstractJDBCDataset(JDBCDataSet jdbcDataset) {
this(jdbcDataset.toSpagoBiDataSet());
}

public AbstractJDBCDataset(SpagoBiDataSet dataSetConfig) {
protected AbstractJDBCDataset(SpagoBiDataSet dataSetConfig) {
super(dataSetConfig);

setDataProxy(new JDBCDataProxy());
Expand All @@ -108,9 +106,11 @@ public AbstractJDBCDataset(SpagoBiDataSet dataSetConfig) {
JSONObject jsonConf = ObjectUtils.toJSONObject(config);
setQuery((jsonConf.get(QUERY) != null) ? jsonConf.get(QUERY).toString() : "");
setQueryScript((jsonConf.get(QUERY_SCRIPT) != null) ? jsonConf.get(QUERY_SCRIPT).toString() : "");
setQueryScriptLanguage((jsonConf.get(QUERY_SCRIPT_LANGUAGE) != null) ? jsonConf.get(QUERY_SCRIPT_LANGUAGE).toString() : "");
setQueryScriptLanguage(
(jsonConf.get(QUERY_SCRIPT_LANGUAGE) != null) ? jsonConf.get(QUERY_SCRIPT_LANGUAGE).toString()
: "");
} catch (Exception e) {
logger.error("Error while defining dataset configuration. Error:", e);
LOGGER.error("Error while defining dataset configuration.", e);
throw new SpagoBIRuntimeException("Error while defining dataset configuration", e);
}

Expand All @@ -130,7 +130,7 @@ public void setUserProfileAttributes(Map userProfile) {
try {
schema = (String) userProfile.get(dataSource.getSchemaAttribute());
((JDBCDataProxy) dataProxy).setSchema(schema);
logger.debug("Set UP Schema=" + schema);
LOGGER.debug("Set UP Schema={}", schema);
} catch (Throwable t) {
throw new SpagoBIRuntimeException("An error occurred while reading schema name from user profile", t);
}
Expand All @@ -156,7 +156,7 @@ public SpagoBiDataSet toSpagoBiDataSet() {
jsonConf.put(QUERY_SCRIPT_LANGUAGE, (getQueryScriptLanguage() == null) ? "" : getQueryScriptLanguage());
toReturn.setConfiguration(jsonConf.toString());
} catch (Exception e) {
logger.error("Error while defining dataset configuration. Error:", e);
LOGGER.error("Error while defining dataset configuration. Error:", e);
throw new SpagoBIRuntimeException("Error while defining dataset configuration. Error:", e);
}

Expand All @@ -174,9 +174,11 @@ public JDBCDataProxy getDataProxy() {
dataProxy = getDataProxy();
}

if (!(dataProxy instanceof JDBCDataProxy) && !(dataProxy instanceof JDBCRedShiftDataProxy) && !(dataProxy instanceof JDBCBigQueryDataProxy)
&& !(dataProxy instanceof JDBCSynapseDataProxy) && !(dataProxy instanceof JDBCSpannerDataProxy)) {
throw new RuntimeException("DataProxy cannot be of type [" + dataProxy.getClass().getName() + "] in JDBCDataSet");
if (!(dataProxy instanceof JDBCDataProxy) && !(dataProxy instanceof JDBCRedShiftDataProxy)
&& !(dataProxy instanceof JDBCBigQueryDataProxy) && !(dataProxy instanceof JDBCSynapseDataProxy)
&& !(dataProxy instanceof JDBCSpannerDataProxy)) {
throw new RuntimeException(
"DataProxy cannot be of type [" + dataProxy.getClass().getName() + "] in JDBCDataSet");
}

return (JDBCDataProxy) dataProxy;
Expand All @@ -199,7 +201,7 @@ public IMetaData getMetadata() {
DatasetMetadataParser dsp = new DatasetMetadataParser();
metadata = dsp.xmlToMetadata(getDsMetadata());
} catch (Exception e) {
logger.error("Error loading the metadata", e);
LOGGER.error("Error loading the metadata", e);
throw new SpagoBIEngineRuntimeException("Error loading the metadata", e);
}
return metadata;
Expand All @@ -215,73 +217,81 @@ public IDataStore test() {
public IDataSetTableDescriptor persist(String tableName, IDataSource dataSource) {
IDataSource datasetDataSource = getDataSource();
if (datasetDataSource.getLabel().equals(dataSource.getLabel())) {
logger.debug("Specified datasource is the dataset's datasource; using CREATE TABLE AS SELECT tecnique");
LOGGER.debug("Specified datasource is the dataset's datasource; using CREATE TABLE AS SELECT tecnique");
try {
List<String> fields = this.getFieldsList();
return TemporaryTableManager.createTable(fields, (String) query, tableName, getDataSource());
} catch (Exception e) {
logger.error("Error peristing the temporary table", e);
LOGGER.error("Error peristing the temporary table", e);
throw new SpagoBIEngineRuntimeException("Error peristing the temporary table", e);
}
} else {
logger.debug("Specified datasource is NOT the dataset's datasource");
LOGGER.debug("Specified datasource is NOT the dataset's datasource");
return super.persist(tableName, dataSource);
}
}

public static String encapsulateColumnName(String columnName, IDataSource dataSource) {
logger.debug("IN");
LOGGER.debug("IN");
try {
String toReturn = columnName;
if (columnName != null) {
String aliasDelimiter = TemporaryTableManager.getAliasDelimiter(dataSource);
logger.debug("Alias delimiter is [" + aliasDelimiter + "]");
LOGGER.debug("Alias delimiter is [{}]", aliasDelimiter);
if (!columnName.startsWith(aliasDelimiter) || !columnName.endsWith(aliasDelimiter)) {
toReturn = aliasDelimiter + columnName + aliasDelimiter;
}
}
logger.debug("OUT: returning " + toReturn);
LOGGER.debug("OUT: returning {}", toReturn);
return toReturn;
} catch (DataBaseException e) {
throw new SpagoBIRuntimeException(e);
}
}

public static String substituteStandardWithDatasourceDelimiter(String columnName, IDataSource dataSource) throws DataBaseException {
logger.debug("IN");
public static String substituteStandardWithDatasourceDelimiter(String columnName, IDataSource dataSource)
throws DataBaseException {
LOGGER.debug("IN");
if (columnName != null) {
logger.debug("Column name is [" + columnName + "]");
columnName = columnName.replaceAll(AbstractDataBase.STANDARD_ALIAS_DELIMITER, TemporaryTableManager.getAliasDelimiter(dataSource));
logger.debug("Column name after replacement is [" + columnName + "]");
LOGGER.debug("Column name is [{}]", columnName);
columnName = columnName.replaceAll(AbstractDataBase.STANDARD_ALIAS_DELIMITER,
TemporaryTableManager.getAliasDelimiter(dataSource));
LOGGER.debug("Column name after replacement is [{}]", columnName);
} else {
throw new IllegalArgumentException("THe arguments columnName [" + columnName + "] and dataSource [" + dataSource + "] cannot be null");
throw new IllegalArgumentException(
"THe arguments columnName [" + columnName + "] and dataSource [" + dataSource + "] cannot be null");
}
logger.debug("OUT");
LOGGER.debug("OUT");
return columnName;
}

@Override
public DataIterator iterator() {
logger.debug("IN");
LOGGER.debug("IN");
try {
IMetaData metadata = getMetadata();
QuerableBehaviour querableBehaviour = (QuerableBehaviour) getBehaviour(QuerableBehaviour.class.getName());
String statement = querableBehaviour.getStatement();
logger.debug("Obtained statement [" + statement + "]");
LOGGER.debug("Obtained statement [{}]", statement);
dataProxy.setStatement(statement);
JDBCDataProxy jdbcDataProxy = (JDBCDataProxy) dataProxy;
IDataSource dataSource = jdbcDataProxy.getDataSource();
Assert.assertNotNull(dataSource, "Invalid datasource");
Connection connection = dataSource.getConnection();
Statement stmt = connection.createStatement(ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_READ_ONLY);
connection.setAutoCommit(false); // PostgreSQL requires disabling auto-commit for setFetchSize to work
stmt.setFetchSize(5000);
ResultSet rs = (ResultSet) dataProxy.getData(dataReader, stmt);
DataIterator iterator = new ResultSetIterator(connection, stmt, rs);
return iterator;
try (Connection connection = dataSource.getConnection();
Statement stmt = connection.createStatement(ResultSet.TYPE_FORWARD_ONLY,
ResultSet.CONCUR_READ_ONLY)) {

connection.setAutoCommit(false); // PostgreSQL requires disabling auto-commit for setFetchSize to work
stmt.setFetchSize(5000);

try (ResultSet rs = (ResultSet) dataProxy.getData(dataReader, stmt)) {
return new ResultSetIterator(rs, metadata);
}
}
} catch (ClassNotFoundException | SQLException | NamingException e) {
throw new SpagoBIRuntimeException(e);
} finally {
logger.debug("OUT");
LOGGER.debug("OUT");
}
}

Expand All @@ -308,7 +318,8 @@ public void setParametersMap(Map<String, String> paramValues) throws JSONExcepti
List<JSONObject> parameters = getDataSetParameters();
if (parameters.size() > paramValues.size()) {
String parameterNotValorizedStr = getParametersNotValorized(parameters, paramValues);
throw new ParametersNotValorizedException("The following parameters have no value [" + parameterNotValorizedStr + "]");
throw new ParametersNotValorizedException(
"The following parameters have no value [" + parameterNotValorizedStr + "]");
}

if (paramValues.size() > 0) {
Expand All @@ -332,7 +343,7 @@ private String[] getValuesAsArray(Map<String, String> paramValues, String paramN
String paramValue = paramValues.get(paramName);
String[] values = null;
if (isMultiValue) {
List<String> list = new ArrayList<String>();
List<String> list = new ArrayList<>();
boolean paramValueConsumed = false;
try {
JSONArray jsonArray = new JSONArray(paramValue);
Expand All @@ -347,35 +358,27 @@ private String[] getValuesAsArray(Map<String, String> paramValues, String paramN
list.add(paramValue);
}
values = list.toArray(new String[0]);
if (values != null && values.length == 1 && !values[0].isEmpty()) {
String valuesString = values[0];
// if (valuesString.startsWith("'") && valuesString.endsWith("'")) {
// // patch for KNOWAGE-4600: An error occurs when propagating a driver value with commas through cross navigation.
// // Do nothing, keep values as it is
// } else {
// values = valuesString.split(",");
// }
}
} else {
values = Arrays.asList(paramValue).toArray(new String[0]);
}
return values;
}

private static String getParametersNotValorized(List<JSONObject> parameters, Map<String, String> parametersValues) throws JSONException {
String toReturn = "";
private static String getParametersNotValorized(List<JSONObject> parameters, Map<String, String> parametersValues)
throws JSONException {
StringBuilder toReturn = new StringBuilder("");

for (Iterator<JSONObject> iterator = parameters.iterator(); iterator.hasNext();) {
JSONObject parameter = iterator.next();
String parameterName = parameter.getString("namePar");
if (parametersValues.get(parameterName) == null) {
toReturn += parameterName;
toReturn.append(parameterName);
if (iterator.hasNext()) {
toReturn += ", ";
toReturn.append(", ");
}
}
}
return toReturn;
return toReturn.toString();
}

/**
Expand Down Expand Up @@ -411,30 +414,30 @@ private List<String> encapsulateValues(JSONObject parameter, String[] values) {
if (value.contains("','")) {
value = value.substring(1, value.length() - 1);
String[] valuesArray = value.split("','");
String newValuesFromArray = "";
StringBuilder newValuesFromArray = new StringBuilder("");
for (int i = 0; i < valuesArray.length; i++) {
String temp = valuesArray[i];
if (!delim.isEmpty() && temp.startsWith(delim) && temp.endsWith(delim))
temp = temp.substring(1, temp.length() - 1);
temp = temp.replaceAll("'", "''");
temp = temp.replace("'", "''");
if (i == 0)
newValuesFromArray = (delim + temp + delim);
newValuesFromArray.append(delim + temp + delim);
else
newValuesFromArray = newValuesFromArray + "," + (delim + temp + delim);
newValuesFromArray.append(",").append(delim + temp + delim);

}
newValues.add(newValuesFromArray);
newValues.add(newValuesFromArray.toString());
} else {
if (isString) {
value = value.substring(1, value.length() - 1);
value = value.replaceAll("'", "''");
value = value.replace("'", "''");
}
newValues.add(delim + value + delim);
}
} else {
if (isString) {
// Duplicate single quote to transform it into an escaped SQL single quote
value = value.replaceAll("'", "''");
value = value.replace("'", "''");
}
newValues.add(delim + value + delim);
}
Expand Down
17 changes: 10 additions & 7 deletions knowageutils/src/main/java/it/eng/spagobi/tools/dataset/bo/FlatDataSet.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -57,7 +57,6 @@ public class FlatDataSet extends ConfigurableDataSet {
private IMetaData metadata;

public FlatDataSet() {
super();
}

public FlatDataSet(SpagoBiDataSet dataSetConfig) {
Expand Down Expand Up @@ -161,13 +160,17 @@ public void setConfiguration(String configuration) {
public DataIterator iterator() {
LOGGER.debug("IN");
try {
IMetaData metadata = getMetadata();
String query = "select * from " + this.getTableName();
Connection connection = dataSource.getConnection();
Statement stmt = connection.createStatement();
stmt.setFetchSize(5000);
ResultSet rs = stmt.executeQuery(query);
DataIterator iterator = new ResultSetIterator(connection, stmt, rs);
return iterator;
try (Connection connection = dataSource.getConnection(); Statement stmt = connection.createStatement()) {

connection.setAutoCommit(false); // PostgreSQL requires disabling auto-commit for setFetchSize to work
stmt.setFetchSize(5000);

try (ResultSet rs = stmt.executeQuery(query)) {
return new ResultSetIterator(rs, metadata);
}
}
} catch (Exception e) {
throw new SpagoBIRuntimeException(e);
} finally {
Expand Down
21 changes: 13 additions & 8 deletions knowageutils/src/main/java/it/eng/spagobi/tools/dataset/bo/JDBCPostgreSQLDataSet.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,7 @@
import it.eng.spagobi.tools.dataset.common.datareader.JDBCStandardDataReader;
import it.eng.spagobi.tools.dataset.common.iterator.DataIterator;
import it.eng.spagobi.tools.dataset.common.iterator.ResultSetIterator;
import it.eng.spagobi.tools.dataset.common.metadata.IMetaData;
import it.eng.spagobi.tools.datasource.bo.IDataSource;
import it.eng.spagobi.utilities.assertion.Assert;
import it.eng.spagobi.utilities.exceptions.SpagoBIRuntimeException;
Expand All @@ -47,7 +48,6 @@ public class JDBCPostgreSQLDataSet extends JDBCDataSet {
* Instantiates a new empty JDBC PostgreSQL data set.
*/
public JDBCPostgreSQLDataSet() {
super();
setDataProxy(new JDBCPostgreSQLDataProxy());
setDataReader(createDataReader());
}
Expand All @@ -65,20 +65,25 @@ protected AbstractDataReader createDataReader() {
public DataIterator iterator() {
logger.debug("IN");
try {
IMetaData metadata = getMetadata();
QuerableBehaviour querableBehaviour = (QuerableBehaviour) getBehaviour(QuerableBehaviour.class.getName());
String statement = querableBehaviour.getStatement();
logger.debug("Obtained statement [" + statement + "]");
dataProxy.setStatement(statement);
JDBCDataProxy jdbcDataProxy = (JDBCDataProxy) dataProxy;
IDataSource dataSource = jdbcDataProxy.getDataSource();
Assert.assertNotNull(dataSource, "Invalid datasource");
Connection connection = dataSource.getConnection();
connection.setAutoCommit(false);
Statement stmt = connection.createStatement(ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_READ_ONLY);
stmt.setFetchSize(5000);
ResultSet rs = (ResultSet) dataProxy.getData(dataReader, stmt);
DataIterator iterator = new ResultSetIterator(connection, stmt, rs);
return iterator;
try (Connection connection = dataSource.getConnection();
Statement stmt = connection.createStatement(ResultSet.TYPE_FORWARD_ONLY,
ResultSet.CONCUR_READ_ONLY)) {

connection.setAutoCommit(false); // PostgreSQL requires disabling auto-commit for setFetchSize to work
stmt.setFetchSize(5000);

try (ResultSet rs = (ResultSet) dataProxy.getData(dataReader, stmt)) {
return new ResultSetIterator(rs, metadata);
}
}
} catch (ClassNotFoundException | SQLException | NamingException e) {
throw new SpagoBIRuntimeException(e);
} finally {
Expand Down
Loading

0 comments on commit d2a70fc

Please sign in to comment.