Skip to content

Commit

Permalink
Merge pull request #536 from wcmc-its/development
Browse files Browse the repository at this point in the history 
Development
  • Loading branch information
@paulalbert1
paulalbert1 authored Dec 3, 2024
2 parents d897bfa + 1d3c39e commit 7415e99
Show file tree
Hide file tree
Showing 9 changed files with 236 additions and 135 deletions.
2 changes: 1 addition & 1 deletion pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -315,7 +315,7 @@
<dependency>
<groupId>edu.cornell.weill.reciter</groupId>
<artifactId>reciter-article-model</artifactId>
<version>2.0.24</version>
<version>2.0.26</version>
</dependency>
<dependency>
<groupId>edu.cornell.weill.reciter</groupId>
Expand Down
25 changes: 25 additions & 0 deletions ...main/java/reciter/algorithm/article/score/predictor/NeuralNetworkModelArticlesScorer.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,8 @@
import java.io.File;
import java.io.IOException;
import java.io.InputStreamReader;
import java.nio.file.Files;
import java.nio.file.Path;

import org.json.JSONArray;
import org.slf4j.Logger;
Expand Down Expand Up @@ -87,4 +89,27 @@ public JSONArray executeArticleScorePredictor(String category, String articleSco
log.info(stopWatch.getId() + " took " + stopWatch.getTotalTimeSeconds() + "s");
return null;
}
public void deleteFile(Path filePath) {
try {
File file = filePath.toFile();
if (file.exists()) {

if (!file.setWritable(true)) {
log.info("Failed to set write permission for the file."+filePath);
}

if (Files.deleteIfExists(filePath)) {
log.info("File deleted successfully."+filePath);
} else {
log.info("File deletion failed."+filePath);
}

} else {
log.info("File does not exist: " + filePath);
}
} catch (Exception e) {
log.error("An error occurred while deleting the file path " + filePath + ": " + e.getMessage());
}

}
}
68 changes: 42 additions & 26 deletions src/main/java/reciter/algorithm/cluster/article/scorer/ReCiterArticleScorer.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,6 @@
import java.util.Objects;
import java.util.Optional;
import java.util.Properties;
import java.util.Set;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
Expand All @@ -36,9 +35,7 @@

import reciter.algorithm.article.score.predictor.NeuralNetworkModelArticlesScorer;
import reciter.algorithm.evidence.StrategyContext;
import reciter.algorithm.evidence.article.ReCiterArticleStrategyContext;
import reciter.algorithm.evidence.article.RemoveReCiterArticleStrategyContext;
import reciter.algorithm.evidence.feedback.targetauthor.TargetAuthorFeedbackStrategyContext;
import reciter.algorithm.evidence.targetauthor.TargetAuthorStrategyContext;
import reciter.algorithm.evidence.targetauthor.affiliation.AffiliationStrategyContext;
import reciter.algorithm.evidence.targetauthor.affiliation.strategy.CommonAffiliationStrategy;
Expand Down Expand Up @@ -334,6 +331,10 @@ public void runArticleScorer(List<ReCiterArticle> reCiterArticles, Identity iden
}
boolean allTasksCompleted = true;
// Print execution times from futures

/*reCiterArticles.forEach(article -> System.out.println("grant Evidence*************"+ article.getGenderEvidence()
+ "\ngrant Evidece totalScore*****************" + article.getGrantEvidenceTotalScore() + "\nGrant List***********" + article.getGrantList()));*/

for (Future<?> future : futures) {
try {
future.get(); // Ensure all tasks are completed
Expand All @@ -352,7 +353,6 @@ public void runArticleScorer(List<ReCiterArticle> reCiterArticles, Identity iden
public List<ReCiterArticle> executePythonScriptForArticleIdentityTotalScore(List<ReCiterArticle> reCiterArticles, Identity identity) {

slf4jLogger.info("articles Size :", reCiterArticles.size());


List<ReCiterArticleFeedbackIdentityScore> articleIdentityScore = reCiterArticles.stream()
.map(article -> {
Expand Down Expand Up @@ -380,13 +380,19 @@ public List<ReCiterArticle> executePythonScriptForArticleIdentityTotalScore(List
String identityS3BucketName = getProperty("aws.s3.feedback.score.bucketName");

try {
NeuralNetworkModelArticlesScorer nnmodel = new NeuralNetworkModelArticlesScorer();
if(isS3UploadRequired)
{
File jsonFile = new File(fileName);

// Write the User object to the JSON file
objectMapper.writeValue(jsonFile, articleIdentityScore);
uploadJsonFileIntoS3(fileName, jsonFile);
boolean uploadJsonFileIntoS3 = uploadJsonFileIntoS3(fileName, jsonFile);
if(uploadJsonFileIntoS3) {

nnmodel.deleteFile(jsonFile.toPath());
slf4jLogger.info("File deleted successfully: " + jsonFile);
}

}
else
Expand All @@ -395,8 +401,7 @@ public List<ReCiterArticle> executePythonScriptForArticleIdentityTotalScore(List
objectMapper.writeValue(jsonFile,articleIdentityScore);
}
String isS3UploadRequiredString = Boolean.toString(isS3UploadRequired);
NeuralNetworkModelArticlesScorer nnmodel = new NeuralNetworkModelArticlesScorer();
JSONArray articlesIdentityScoreTotal = nnmodel.executeArticleScorePredictor("Identity Score", "identityOnlyScoreArticles.py",fileName,identityS3BucketName,isS3UploadRequiredString);
JSONArray articlesIdentityScoreTotal = nnmodel.executeArticleScorePredictor("Identity Score", "identityOnlyScoreArticles.py",fileName,identityS3BucketName,isS3UploadRequiredString);
return mapAuthorshipLikelihoodScore(reCiterArticles, articlesIdentityScoreTotal);


Expand Down Expand Up @@ -569,7 +574,7 @@ private Properties PropertiesLoader(String fileName) {
private String getProperty(String key) {
return properties.getProperty(key);
}
private void uploadJsonFileIntoS3(String keyName,File file)
private boolean uploadJsonFileIntoS3(String keyName,File file)
{
String FeedbackScoreBucketName = getProperty("aws.s3.feedback.score.bucketName");

Expand All @@ -581,30 +586,41 @@ private void uploadJsonFileIntoS3(String keyName,File file)
.withCredentials(new DefaultAWSCredentialsProviderChain())
.withRegion(System.getenv("AWS_REGION"))
.build();
if(s3.doesBucketExistV2(FeedbackScoreBucketName))
{

slf4jLogger.info("Uploading files to S3 bucket ",FeedbackScoreBucketName);
PutObjectRequest putObjectRequest = new PutObjectRequest(FeedbackScoreBucketName.toLowerCase(), keyName, file);

// Optionally, set metadata
ObjectMetadata metadata = new ObjectMetadata();
metadata.setContentType("application/json");
putObjectRequest.setMetadata(metadata);


try{
s3.putObject(putObjectRequest);
slf4jLogger.info("CSV file uploaded successfully to S3 bucket: " + FeedbackScoreBucketName);
}
catch(AmazonServiceException e) {
// The call was transmitted successfully, but Amazon S3 couldn't process
// it, so it returned an error response.
slf4jLogger.error(e.getErrorMessage());
slf4jLogger.info("Uploading files to S3 bucket ",FeedbackScoreBucketName);
PutObjectRequest putObjectRequest = new PutObjectRequest(FeedbackScoreBucketName.toLowerCase(), keyName, file);

// Optionally, set metadata
ObjectMetadata metadata = new ObjectMetadata();
metadata.setContentType("application/json");
putObjectRequest.setMetadata(metadata);


try{
s3.putObject(putObjectRequest);
slf4jLogger.info("CSV file uploaded successfully to S3 bucket: " + FeedbackScoreBucketName);
}
catch(AmazonServiceException e) {
// The call was transmitted successfully, but Amazon S3 couldn't process
// it, so it returned an error response.
slf4jLogger.error(e.getErrorMessage());
return false;
}
return true;
}
else
{
slf4jLogger.error("S3 bucket does not exist: " + FeedbackScoreBucketName);
return false;
}

} catch (Exception e) {
e.printStackTrace();
return false;
}
}
}
private Future<?> submitAndLogTime(String category, ExecutorService executorService,
StrategyContext context,List<ReCiterArticle> reCiterArticles, Identity identity)
{
Expand Down
23 changes: 8 additions & 15 deletions ...dence/targetauthor/feedback/journalsubfield/strategy/JournalSubFieldFeedbackStrategy.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,12 +3,15 @@
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
import java.util.function.Function;
import java.util.stream.Collectors;
import java.util.stream.Stream;

import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
Expand All @@ -25,9 +28,12 @@
public class JournalSubFieldFeedbackStrategy extends AbstractTargetAuthorFeedbackStrategy {

private static final Logger slf4jLogger = LoggerFactory.getLogger(JournalSubFieldFeedbackStrategy.class);
//Map<String, List<ReCiterArticleFeedbackScore>> feedbackJournalsSubFieldMap = null;

private Map<String, List<ReCiterArticleFeedbackScore>> feedbackJournalsSubFieldMap = new HashMap<>();

Set<String> filterJournalSubFields = Stream.of("General Science & Technology")
.collect(Collectors.toSet());

ScienceMetrixService scienceMetrixService = ApplicationContextHolder.getContext()
.getBean(ScienceMetrixService.class);

Expand All @@ -38,19 +44,6 @@ public double executeFeedbackStrategy(ReCiterArticle reCiterArticle, Identity id

}

/*private String retrieveJournalSubField(MedlineCitationJournalISSN medlineCitationJournalIssn) {
if(medlineCitationJournalIssn!=null && medlineCitationJournalIssn.getIssn()!=null && !medlineCitationJournalIssn.getIssn().isEmpty())
{
ScienceMetrix scienceMetrix = scienceMetrixService.findByIssn(medlineCitationJournalIssn.getIssn());
if (scienceMetrix == null)
scienceMetrix = scienceMetrixService.findByEissn(medlineCitationJournalIssn.getIssn());
if (scienceMetrix != null && scienceMetrix.getScienceMetrixSubfield()!=null &&
!scienceMetrix.getScienceMetrixSubfield().equalsIgnoreCase(""))
return scienceMetrix.getScienceMetrixSubfield();
}
return null;
}*/

@Override
public double executeFeedbackStrategy(List<ReCiterArticle> reCiterArticles, Identity identity) {
Expand Down Expand Up @@ -122,7 +115,7 @@ private ReCiterArticleFeedbackScore processArticle(ReCiterArticle article, Medli
Map<String, Long> rejectedArticlesCountByJournalSubField) {

String journalSubField = retrieveJournalSubField(journalIssn);
if (journalSubField == null || journalSubField.isEmpty()) {
if (journalSubField == null || journalSubField.isEmpty() || filterJournalSubFields.contains(journalSubField)) {
return null; // Skip if no valid subfield
}

Expand Down
Loading

0 comments on commit 7415e99

Please sign in to comment.