From 141e24bab53121104c6228fd0a4b62ab62038c7c Mon Sep 17 00:00:00 2001 From: Afsah Syeda Date: Fri, 6 Oct 2023 11:57:34 +0530 Subject: [PATCH] feat(Release):Upload Source Code Attachment to Releases through a Scheduled Service Signed-off-by: afsahsyeda --- .../db/ComponentDatabaseHandler.java | 155 +++++++++++++++++- .../datahandler/db/ComponentRepository.java | 4 + .../sw360/components/ComponentHandler.java | 46 +++--- .../schedule/service/ScheduleHandler.java | 3 + .../schedule/timer/ScheduleConstants.java | 5 + .../sw360/portal/common/PortalConstants.java | 7 +- .../portlets/admin/ScheduleAdminPortlet.java | 21 +++ .../html/admin/scheduleAdmin/view.jsp | 41 +++++ .../resources/content/Language.properties | 3 + .../resources/content/Language_ja.properties | 1 + .../resources/content/Language_vi.properties | 1 + .../resources/content/Language_zh.properties | 1 + .../commonIO/AttachmentFrontendUtils.java | 6 +- .../sw360/datahandler/common/CommonUtils.java | 18 +- .../datahandler/common/SW360Constants.java | 6 + .../datahandler/thrift/ThriftClients.java | 1 + .../src/main/thrift/components.thrift | 5 + 17 files changed, 297 insertions(+), 27 deletions(-) diff --git a/backend/src-common/src/main/java/org/eclipse/sw360/datahandler/db/ComponentDatabaseHandler.java b/backend/src-common/src/main/java/org/eclipse/sw360/datahandler/db/ComponentDatabaseHandler.java index 656bf2adc6..0a404e5150 100644 --- a/backend/src-common/src/main/java/org/eclipse/sw360/datahandler/db/ComponentDatabaseHandler.java +++ b/backend/src-common/src/main/java/org/eclipse/sw360/datahandler/db/ComponentDatabaseHandler.java @@ -15,6 +15,7 @@ import com.google.common.collect.*; import org.eclipse.sw360.common.utils.BackendUtils; +import org.eclipse.sw360.commonIO.AttachmentFrontendUtils; import org.eclipse.sw360.components.summary.SummaryType; import org.eclipse.sw360.datahandler.cloudantclient.DatabaseConnectorCloudant; import org.eclipse.sw360.datahandler.common.CommonUtils; @@ -68,11 +69,16 @@ import org.eclipse.sw360.spdx.SpdxBOMImporterSink; import org.jetbrains.annotations.NotNull; import java.io.File; +import java.io.FileInputStream; import java.io.IOException; import java.io.InputStream; import org.spdx.library.InvalidSPDXAnalysisException; + +import java.net.HttpURLConnection; import java.net.MalformedURLException; +import java.net.URL; import java.nio.ByteBuffer; +import java.nio.file.*; import java.util.*; import java.util.concurrent.TimeUnit; import java.util.function.Function; @@ -112,6 +118,7 @@ public class ComponentDatabaseHandler extends AttachmentAwareDatabaseHandler { private static final String NO_RELEASE = "Don't have Release created!"; private static final List listComponentName = new ArrayList<>(); private static final Map mapReleaseName = new HashMap<>(); + public static final List formats = new ArrayList<>(Arrays.asList(SW360Constants.URL_FORMATS.split(","))); /** * Connection to the couchDB database @@ -124,7 +131,7 @@ public class ComponentDatabaseHandler extends AttachmentAwareDatabaseHandler { private final PackageRepository packageRepository; private DatabaseHandlerUtil dbHandlerUtil; private BulkDeleteUtil bulkDeleteUtil; - + private final AttachmentConnector attachmentConnector; private SvmConnector svmConnector; private final SpdxDocumentDatabaseHandler spdxDocumentDatabaseHandler; @@ -179,7 +186,7 @@ public ComponentDatabaseHandler(Supplier httpClient, String dbNa attachmentConnector = new AttachmentConnector(httpClient, attachmentDbName, durationOf(30, TimeUnit.SECONDS)); DatabaseConnectorCloudant dbChangeLogs = new DatabaseConnectorCloudant(httpClient, DatabaseSettings.COUCH_DB_CHANGE_LOGS); this.dbHandlerUtil = new DatabaseHandlerUtil(dbChangeLogs); - + this.bulkDeleteUtil = new BulkDeleteUtil(this, componentRepository, releaseRepository, projectRepository, moderator, releaseModerator, attachmentConnector, attachmentDatabaseHandler, dbHandlerUtil); @@ -393,6 +400,11 @@ public Component getAccessibleComponent(String id, User user) throws SW360Except return component; } + //Used by scheduled upload service to get releases of a component without user info + public Release getRelease(String id) { + return releaseRepository.get(id); + } + public Release getRelease(String id, User user) throws SW360Exception { return getRelease(id, user, null); } @@ -1460,11 +1472,11 @@ public void recomputeReleaseDependentFields(Component component, String skipThis updateReleaseDependentFieldsForComponent(component, containedRelease); } } - + public BulkOperationNode deleteBulkRelease(String releaseId, User user, boolean isPreview) throws SW360Exception { return bulkDeleteUtil.deleteBulkRelease(releaseId, user, isPreview); } - + public BulkDeleteUtil getBulkDeleteUtil() { return bulkDeleteUtil; } @@ -2000,6 +2012,11 @@ public Map getAllComponentsIdMap() { return ThriftUtils.getIdMap(components); } + public List getAllComponentsWithVCS() { + final List components = componentRepository.getComponentsByVCS(); + return components; + } + @NotNull private List iterateReleaseRelationShips(Map relations, String parentNodeId, Deque visitedIds) { List out = new ArrayList<>(); @@ -3039,4 +3056,134 @@ private ReleaseNode getReleaseNodes(ReleaseNode releaseNode, User user) { } return releaseNode; } + + public RequestStatus uploadSourceCodeAttachmentToReleases() { + List components = getAllComponentsWithVCS(); + Set releasesWithoutSRC = new HashSet<>(); + Set updateReleases = new HashSet<>(); + log.info(String.format("SRC Upload: Found %d components with VCS", components.size())); + + components.forEach(c -> { + String VCS = c.getVcs(); + log.info(String.format("SRC Upload: %s %s", c.getId(), VCS)); + if (isValidURL(VCS)) { + for (String r_id : c.getReleaseIds()) { + boolean isUploaded = false; + Release r = getRelease(r_id); + + if (r.getClearingState() == ClearingState.NEW_CLEARING) { + List sourceAttachments = (r.getAttachments() != null) ? r.getAttachments().stream() + .filter(attachment -> AttachmentType.SOURCE.equals(attachment.getAttachmentType())) + .collect(Collectors.toList()) : Collections.emptyList(); + + if (sourceAttachments.size() == 0) { + releasesWithoutSRC.add(r.getId()); + String version = r.getVersion(); + Release originalReleaseData = r.deepCopy(); + + for (String format : formats) { + String downloadURL = String.format(format, c.getVcs(), version); + if (isValidURL(downloadURL)) { + try { + String destinationDirectory = SW360Constants.SRC_ATTACHMENT_DOWNLOAD_LOCATION; + File file = downloadFile(downloadURL, destinationDirectory); + Attachment attachment = new Attachment() + .setAttachmentType(AttachmentType.SOURCE); + Set src_attachment = new HashSet<>(); + src_attachment.add(uploadAttachment(file, attachment)); + r.setAttachments(src_attachment); + r.setSourceCodeDownloadurl(downloadURL); + releaseRepository.update(r); + isUploaded = true; + updateReleases.add(r.getId()); + // Delete the SRC zip file after the release is updated + file.delete(); + break; + } catch (IOException | TException e) { + log.error( + "SRC Upload: Error while downloading the source code zip file for release:" + + r.getId() + " " + e); + } + } + } + if (isUploaded) { + dbHandlerUtil.addChangeLogs(r, originalReleaseData, + SW360Constants.SRC_ATTACHMENT_UPLOADER_EMAIL, Operation.UPDATE, + attachmentConnector, Lists.newArrayList(), null, null); + } + } + } + } + } + }); + if (updateReleases.size() == releasesWithoutSRC.size()) { + log.info(String.format("SRC Upload: updated %d releases", updateReleases.size())); + return RequestStatus.SUCCESS; + } else { + log.error("SRC Upload: Failed to upload SRC attachments for releases: " + + Sets.difference(releasesWithoutSRC, updateReleases)); + return RequestStatus.FAILURE; + } + } + + private boolean isValidURL(String url) { + try { + URL urlObj = new URL(url); + HttpURLConnection connection = (HttpURLConnection) urlObj.openConnection(); + connection.setRequestMethod("HEAD"); + int responseCode = connection.getResponseCode(); + + if (responseCode == HttpURLConnection.HTTP_OK) { + return true; + } else { + return false; + } + } catch (IOException e) { + log.error("Error while checking the validity of the URL " + url, e); + return false; + } + } + + public File downloadFile(String url, String destinationDirectory) throws IOException { + URL fileUrl = new URL(url); + String regex = ".*/([^/]+)/archive/refs/tags/(?:v)?([\\d.]+)\\.zip$"; + String fileName = url.replaceAll(regex, "$1-$2.zip"); + Path destinationPath = Paths.get(destinationDirectory, fileName); + + try (InputStream in = fileUrl.openStream()) { + Files.copy(in, destinationPath, StandardCopyOption.REPLACE_EXISTING); + } + return destinationPath.toFile(); + } + + public Attachment uploadAttachment(File file, Attachment newAttachment) throws IOException, TException { + String fileName = file.getName(); + String contentType = "application/zip"; + final AttachmentContent attachmentContent = makeAttachmentContent(fileName, contentType); + FileInputStream inputStream = new FileInputStream(file); + Attachment attachment = new AttachmentFrontendUtils().uploadAttachmentContent(attachmentContent, inputStream, null); + + attachment.setSha1(attachmentConnector.getSha1FromAttachmentContentId(attachmentContent.getId())); + attachment.setAttachmentType(AttachmentType.SOURCE); + attachment.setCheckStatus(CheckStatus.NOTCHECKED); + attachment.setCreatedComment("Uploaded by the SW360 scheduled service based on the VCS url of the component"); + attachment.setCreatedBy(SW360Constants.SRC_ATTACHMENT_UPLOADER_EMAIL); + return attachment; + } + + private AttachmentContent makeAttachmentContent(String filename, String contentType) { + AttachmentContent attachment = new AttachmentContent() + .setContentType(contentType) + .setFilename(filename) + .setOnlyRemote(false); + return makeAttachmentContent(attachment); + } + + private AttachmentContent makeAttachmentContent(AttachmentContent content) { + try { + return new AttachmentFrontendUtils().makeAttachmentContent(content); + } catch (TException e) { + throw new RuntimeException(e); + } + } } diff --git a/backend/src-common/src/main/java/org/eclipse/sw360/datahandler/db/ComponentRepository.java b/backend/src-common/src/main/java/org/eclipse/sw360/datahandler/db/ComponentRepository.java index 16093c5612..ab65af69ec 100644 --- a/backend/src-common/src/main/java/org/eclipse/sw360/datahandler/db/ComponentRepository.java +++ b/backend/src-common/src/main/java/org/eclipse/sw360/datahandler/db/ComponentRepository.java @@ -321,4 +321,8 @@ public Map> getRecentComponentsSummary(User user result.put(pageData, components); return result; } + + public List getComponentsByVCS() { + return queryView("byVCS"); + } } diff --git a/backend/src/src-components/src/main/java/org/eclipse/sw360/components/ComponentHandler.java b/backend/src/src-components/src/main/java/org/eclipse/sw360/components/ComponentHandler.java index 0a44815522..cecd8b62a6 100644 --- a/backend/src/src-components/src/main/java/org/eclipse/sw360/components/ComponentHandler.java +++ b/backend/src/src-components/src/main/java/org/eclipse/sw360/components/ComponentHandler.java @@ -109,7 +109,7 @@ public int getAccessibleTotalComponentsCount(User user) throws TException { assertUser(user); return handler.getAccessibleTotalComponentsCount(user); } - + @Override public List getReleaseSummary(User user) throws TException { assertUser(user); @@ -123,7 +123,7 @@ public List getAccessibleReleaseSummary(User user) throws TException { return handler.getAccessibleReleaseSummary(user); } - + @Override public Map> getAccessibleReleasesWithPagination(User user, PaginationData pageData) throws TException { assertUser(user); @@ -161,7 +161,7 @@ public List searchReleases(String searchText) throws TException { public List searchAccessibleReleases(String searchText, User user) throws TException { return handler.searchAccessibleReleasesByText(releaseSearchHandler, searchText, user) ; } - + @Override public List searchReleaseByNamePrefix(String name) throws TException { return handler.searchReleaseByNamePrefix(name); @@ -190,7 +190,7 @@ public List getRecentReleases() throws TException { public List getRecentReleasesWithAccessibility(User user) throws TException { return handler.getRecentReleasesWithAccessibility(user); } - + //////////////////////////// // GET INDIVIDUAL OBJECTS // //////////////////////////// @@ -204,7 +204,7 @@ public Component getComponentById(String id, User user) throws TException { handler.addSelectLogs(component, user); return component; } - + @Override public Component getAccessibleComponentById(String id, User user) throws SW360Exception { assertId(id); @@ -262,7 +262,7 @@ public Release getAccessibleReleaseByIdForEdit(String id, User user) throws SW36 return handler.getAccessibleReleaseForEdit(id, user); } - + @Override public List getReleasesByIdsForExport(Set ids) throws TException { assertNotNull(ids); @@ -281,7 +281,7 @@ public List getReleasesWithAccessibilityByIdsForExport(Set ids, assertUser(user); return handler.getDetailedReleasesWithAccessibilityForExport(ids, user); } - + @Override public List getReleasesById(Set ids, User user) throws TException { assertUser(user); @@ -295,7 +295,7 @@ public List getAccessibleReleasesById(Set ids, User user) throw assertNotNull(ids); return handler.getAccessibleReleases(ids, user); } - + @Override public List getFullReleasesById(Set ids, User user) throws TException { assertUser(user); @@ -326,12 +326,12 @@ public List getReleasesFromVendorIds(Set ids) throws TException public List getAccessibleReleasesFromVendorIds(Set ids, User user) throws TException { return handler.getAccessibleReleasesFromVendorIds(ids, user); } - + @Override public Set getReleasesByVendorId(String vendorId) throws TException { return handler.getReleasesByVendorId(vendorId); } - + //////////////////////////// // ADD INDIVIDUAL OBJECTS // //////////////////////////// @@ -367,7 +367,7 @@ public RequestStatus updateComponent(Component component, User user) throws TExc return handler.updateComponent(component, user); } - + @Override public RequestStatus updateComponentWithForceFlag(Component component, User user, boolean forceUpdate) throws TException { assertNotNull(component); @@ -376,7 +376,7 @@ public RequestStatus updateComponentWithForceFlag(Component component, User user return handler.updateComponent(component, user, forceUpdate); } - + @Override public RequestSummary updateComponents(Set components, User user) throws TException { assertUser(user); @@ -384,6 +384,7 @@ public RequestSummary updateComponents(Set components, User user) thr return handler.updateComponents(components, user); } + @Override public RequestStatus updateComponentFromModerationRequest(Component componentAdditions, Component componentDeletions, User user) { return handler.updateComponentFromAdditionsAndDeletions(componentAdditions, componentDeletions, user); } @@ -411,7 +412,7 @@ public RequestStatus updateReleaseWithForceFlag(Release release, User user, bool removeSelfLink(release); return handler.updateRelease(release, user, ThriftUtils.IMMUTABLE_OF_RELEASE, forceUpdate); } - + private void removeSelfLink(Release release) { if(release.releaseIdToRelationship != null && !release.releaseIdToRelationship.isEmpty()) { release.releaseIdToRelationship.remove(release.id); @@ -439,6 +440,7 @@ public RequestSummary updateReleasesDirectly(Set releases, User user) t return handler.updateReleasesDirectly(releases, user); } + @Override public RequestStatus updateReleaseFromModerationRequest(Release releaseAdditions, Release releaseDeletions, User user) { return handler.updateReleaseFromAdditionsAndDeletions(releaseAdditions, releaseDeletions, user); } @@ -473,7 +475,7 @@ public RequestStatus deleteComponentWithForceFlag(String id, User user, boolean return handler.deleteComponent(id, user, forceDelete); } - + @Override public RequestStatus deleteRelease(String id, User user) throws TException { assertUser(user); @@ -489,7 +491,7 @@ public RequestStatus deleteReleaseWithForceFlag(String id, User user, boolean fo return handler.deleteRelease(id, user, forceDelete); } - + @Override public List getReleasesByComponentId(String id, User user) throws TException { assertUser(user); @@ -517,7 +519,7 @@ public Set getUsingComponentsForRelease(String releaseId) throws TExc public Set getUsingComponentsWithAccessibilityForRelease(String releaseId, User user) throws TException { return handler.getUsingComponentsWithAccessibility(releaseId, user); } - + @Override public Set getUsingComponentsForComponent(Set releaseIds) throws TException { return handler.getUsingComponents(releaseIds); @@ -527,7 +529,7 @@ public Set getUsingComponentsForComponent(Set releaseIds) thr public Set getUsingComponentsWithAccessibilityForComponent(Set releaseIds, User user) throws TException { return handler.getUsingComponentsWithAccessibility(releaseIds, user); } - + @Override public Set getComponentsByDefaultVendorId(String defaultVendorId) throws TException { return handler.getComponentsByDefaultVendorId(defaultVendorId); @@ -549,7 +551,7 @@ public Component recomputeReleaseDependentFields(String componentId, User user) assertId(componentId); return handler.updateReleaseDependentFieldsForComponentId(componentId, user); } - + @Override public BulkOperationNode deleteBulkRelease(String releaseId, User user, boolean isPreview) throws SW360Exception { return handler.deleteBulkRelease(releaseId, user, isPreview); @@ -631,7 +633,7 @@ public List getLinkedReleasesWithAccessibility(Map getLinkedReleaseRelations(Map relations) throws TException { return handler.getLinkedReleases(relations); @@ -652,6 +654,11 @@ public RequestStatus updateReleasesWithSvmTrackingFeedback() throws TException { return handler.updateReleasesWithSvmTrackingFeedback(); } + @Override + public RequestStatus uploadSourceCodeAttachmentToReleases() throws TException { + return handler.uploadSourceCodeAttachmentToReleases(); + } + @Override public Map> getDuplicateComponents() throws TException { return handler.getDuplicateComponents(); @@ -711,6 +718,7 @@ public RequestStatus splitComponent(Component srcComponent, Component targetComp return handler.splitComponent(srcComponent, targetComponent, user); } + @Override public List getAllReleasesForUser(User user) throws TException { assertUser(user); return handler.getAllReleases(); diff --git a/backend/src/src-schedule/src/main/java/org/eclipse/sw360/schedule/service/ScheduleHandler.java b/backend/src/src-schedule/src/main/java/org/eclipse/sw360/schedule/service/ScheduleHandler.java index 0cda488361..20b426b4b4 100644 --- a/backend/src/src-schedule/src/main/java/org/eclipse/sw360/schedule/service/ScheduleHandler.java +++ b/backend/src/src-schedule/src/main/java/org/eclipse/sw360/schedule/service/ScheduleHandler.java @@ -86,6 +86,9 @@ public RequestSummary scheduleService(String serviceName) throws TException { case ThriftClients.IMPORT_DEPARTMENT_SERVICE: successSync = wrapSupplierException(() -> thriftClients.makeUserClient().importDepartmentSchedule(), serviceName); break; + case ThriftClients.SRC_UPLOAD_SERVICE: + successSync = wrapSupplierException(() -> thriftClients.makeComponentClient().uploadSourceCodeAttachmentToReleases(), serviceName); + break; default: log.error("Could not schedule service: " + serviceName + ". Reason: service is not registered in ThriftClients."); } diff --git a/backend/src/src-schedule/src/main/java/org/eclipse/sw360/schedule/timer/ScheduleConstants.java b/backend/src/src-schedule/src/main/java/org/eclipse/sw360/schedule/timer/ScheduleConstants.java index 5ddff2c53b..5157d97266 100644 --- a/backend/src/src-schedule/src/main/java/org/eclipse/sw360/schedule/timer/ScheduleConstants.java +++ b/backend/src/src-schedule/src/main/java/org/eclipse/sw360/schedule/timer/ScheduleConstants.java @@ -52,6 +52,10 @@ private ScheduleConstants(){} public static final String SVM_TRACKING_FEEDBACK_INTERVAL_PROPERTY_NAME = "schedule.trackingfeedback.interval.seconds"; public static final String SVM_TRACKING_FEEDBACK_OFFSET_DEFAULT = (4*60*60) + "" ; // default 04:00 am, in seconds public static final String SVM_TRACKING_FEEDBACK_INTERVAL_DEFAULT = (24*60*60)+"" ; // default 24h, in seconds + public static final String SRC_UPLOAD_SERVICE_OFFSET_PROPERTY_NAME = "schedule.srcupload.firstOffset.seconds"; + public static final String SRC_UPLOAD_SERVICE_OFFSET_DEFAULT = (22*60*60) + ""; //default 10:00 pm, in seconds + public static final String SRC_UPLOAD_SERVICE_INTERVAL_PROPERTY_NAME = "schedule.srcupload.interval.seconds"; + public static final String SRC_UPLOAD_SERVICE_INTERVAL_DEFAULT = (24*60*60)+"" ; // default 24h, in seconds; public static final String DELETE_ATTACHMENT_OFFSET_DEFAULT = "0"; // default 00:00 am, in seconds public static final String DELETE_ATTACHMENT_INTERVAL_DEFAULT = (24*60*60) + "" ; // default 24h, in seconds @@ -75,6 +79,7 @@ private ScheduleConstants(){} loadScheduledServiceProperties(props, ThriftClients.SVMMATCH_SERVICE, SVMMATCH_OFFSET_PROPERTY_NAME, SVMMATCH_OFFSET_DEFAULT, SVMMATCH_INTERVAL_PROPERTY_NAME, SVMMATCH_INTERVAL_DEFAULT); loadScheduledServiceProperties(props, ThriftClients.SVM_LIST_UPDATE_SERVICE, SVM_LIST_UPDATE_OFFSET_PROPERTY_NAME, SVM_LIST_UPDATE_OFFSET_DEFAULT, SVM_LIST_UPDATE_INTERVAL_PROPERTY_NAME, SVM_LIST_UPDATE_INTERVAL_DEFAULT); loadScheduledServiceProperties(props, ThriftClients.SVM_TRACKING_FEEDBACK_SERVICE, SVM_TRACKING_FEEDBACK_OFFSET_PROPERTY_NAME, SVM_TRACKING_FEEDBACK_OFFSET_DEFAULT, SVM_TRACKING_FEEDBACK_INTERVAL_PROPERTY_NAME, SVM_TRACKING_FEEDBACK_INTERVAL_DEFAULT); + loadScheduledServiceProperties(props, ThriftClients.SRC_UPLOAD_SERVICE, SRC_UPLOAD_SERVICE_OFFSET_PROPERTY_NAME, SRC_UPLOAD_SERVICE_OFFSET_DEFAULT, SRC_UPLOAD_SERVICE_INTERVAL_PROPERTY_NAME, SRC_UPLOAD_SERVICE_INTERVAL_DEFAULT); String autostartServicesString = props.getProperty(AUTOSTART_PROPERTY_NAME, ""); autostartServices = autostartServicesString.split(","); diff --git a/frontend/sw360-portlet/src/main/java/org/eclipse/sw360/portal/common/PortalConstants.java b/frontend/sw360-portlet/src/main/java/org/eclipse/sw360/portal/common/PortalConstants.java index 48e40e3cbd..a81f15e9f0 100644 --- a/frontend/sw360-portlet/src/main/java/org/eclipse/sw360/portal/common/PortalConstants.java +++ b/frontend/sw360-portlet/src/main/java/org/eclipse/sw360/portal/common/PortalConstants.java @@ -464,6 +464,11 @@ public class PortalConstants { public static final String DELETE_ATTACHMENT_INTERVAL = "deleteAttachmentInterval"; public static final String DELETE_ATTACHMENT_NEXT_SYNC = "deleteAttachmentNextSync"; + public static final String SRC_UPLOAD_SERVICE_IS_SCHEDULED = "srcUploadIsScheduled"; + public static final String SRC_UPLOAD_SERVICE_OFFSET = "srcUploadOffset"; + public static final String SRC_UPLOAD_SERVICE_INTERVAL = "srcUploadInterval"; + public static final String SRC_UPLOAD_SERVICE_NEXT_SYNC = "srcUploadNextSync"; + public static final String DEPARTMENT_IS_SCHEDULED = "departmentIsScheduled"; public static final String DEPARTMENT_OFFSET = "departmentOffset"; public static final String DEPARTMENT_INTERVAL = "departmentInterval"; @@ -766,7 +771,7 @@ public class PortalConstants { public static final String USER_CLIENT_ID_KEY = "userClientIdKey"; public static final String USER_CLIENT_ID_ACCESS_VALUE = "userClientIdAccessValue"; public static final String USER_CLIENT_ID_NAME_VALUE = "userClientIdNameValue"; - + // Bulk deleting public static final String BULK_DELETING_RESULT_REMAINED = "Remained"; public static final String BULK_DELETING_RESULT_DELETED = "Deleted"; diff --git a/frontend/sw360-portlet/src/main/java/org/eclipse/sw360/portal/portlets/admin/ScheduleAdminPortlet.java b/frontend/sw360-portlet/src/main/java/org/eclipse/sw360/portal/portlets/admin/ScheduleAdminPortlet.java index 01a840ac00..c15d84226a 100644 --- a/frontend/sw360-portlet/src/main/java/org/eclipse/sw360/portal/portlets/admin/ScheduleAdminPortlet.java +++ b/frontend/sw360-portlet/src/main/java/org/eclipse/sw360/portal/portlets/admin/ScheduleAdminPortlet.java @@ -72,6 +72,7 @@ private void prepareStandardView(RenderRequest request, RenderResponse response) putServiceAttributesInRequest(request, user, scheduleClient, ThriftClients.SVMMATCH_SERVICE, PortalConstants.SVMMATCH_IS_SCHEDULED, PortalConstants.SVMMATCH_OFFSET, PortalConstants.SVMMATCH_INTERVAL, PortalConstants.SVMMATCH_NEXT_SYNC); putServiceAttributesInRequest(request, user, scheduleClient, ThriftClients.SVM_LIST_UPDATE_SERVICE, PortalConstants.SVM_LIST_UPDATE_IS_SCHEDULED, PortalConstants.SVM_LIST_UPDATE_OFFSET, PortalConstants.SVM_LIST_UPDATE_INTERVAL, PortalConstants.SVM_LIST_UPDATE_NEXT_SYNC); putServiceAttributesInRequest(request, user, scheduleClient, ThriftClients.SVM_TRACKING_FEEDBACK_SERVICE, PortalConstants.SVM_TRACKING_FEEDBACK_IS_SCHEDULED, PortalConstants.SVM_TRACKING_FEEDBACK_OFFSET, PortalConstants.SVM_TRACKING_FEEDBACK_INTERVAL, PortalConstants.SVM_TRACKING_FEEDBACK_NEXT_SYNC); + putServiceAttributesInRequest(request, user, scheduleClient, ThriftClients.SRC_UPLOAD_SERVICE, PortalConstants.SRC_UPLOAD_SERVICE_IS_SCHEDULED, PortalConstants.SRC_UPLOAD_SERVICE_OFFSET, PortalConstants.SRC_UPLOAD_SERVICE_INTERVAL, PortalConstants.SRC_UPLOAD_SERVICE_NEXT_SYNC); int offsetInSeconds = scheduleClient.getFirstRunOffset(ThriftClients.CVESEARCH_SERVICE); request.setAttribute(PortalConstants.CVESEARCH_OFFSET, CommonUtils.formatTime(offsetInSeconds)); int intervalInSeconds = scheduleClient.getInterval(ThriftClients.CVESEARCH_SERVICE); @@ -319,4 +320,24 @@ public void triggerCveSearch(ActionRequest request, ActionResponse response) thr e.printStackTrace(); } } + + @UsedAsLiferayAction + public void triggeSrcUpload(ActionRequest request, ActionResponse response) throws PortletException, IOException { + try { + RequestStatus requestStatus = new ThriftClients().makeComponentClient().uploadSourceCodeAttachmentToReleases(); + setSessionMessage(request, requestStatus, "Task", "performe"); + } catch (TException e) { + log.error(e); + } + } + + @UsedAsLiferayAction + public void scheduleSrcUpload(ActionRequest request, ActionResponse response) throws PortletException, IOException { + scheduleService(ThriftClients.SRC_UPLOAD_SERVICE, request); + } + + @UsedAsLiferayAction + public void unscheduleSrcUpload(ActionRequest request, ActionResponse response) throws PortletException, IOException { + unscheduleService(ThriftClients.SRC_UPLOAD_SERVICE, request); + } } diff --git a/frontend/sw360-portlet/src/main/resources/META-INF/resources/html/admin/scheduleAdmin/view.jsp b/frontend/sw360-portlet/src/main/resources/META-INF/resources/html/admin/scheduleAdmin/view.jsp index 0e7e671e3d..85dfba3d39 100644 --- a/frontend/sw360-portlet/src/main/resources/META-INF/resources/html/admin/scheduleAdmin/view.jsp +++ b/frontend/sw360-portlet/src/main/resources/META-INF/resources/html/admin/scheduleAdmin/view.jsp @@ -17,6 +17,10 @@ + + + + @@ -94,6 +98,15 @@ + + + + + + + + +
@@ -133,6 +146,31 @@
+
+
+

+ + + + + + + + + + + + + +
${srcUploadOffset} (hh:mm:ss)
${srcUploadInterval} (hh:mm:ss)
${srcUploadNextSync}
+
+
+ + +
+
+
+
@@ -274,6 +312,9 @@
+
+ +
diff --git a/frontend/sw360-portlet/src/main/resources/content/Language.properties b/frontend/sw360-portlet/src/main/resources/content/Language.properties index 338879337e..8a1b1e2538 100644 --- a/frontend/sw360-portlet/src/main/resources/content/Language.properties +++ b/frontend/sw360-portlet/src/main/resources/content/Language.properties @@ -139,6 +139,7 @@ cancel=Cancel cancel.all.scheduled.tasks=Cancel all Scheduled Tasks cancel.cve.service=Cancel CVE service cancel.scheduled.attachment.deletion.from.local.fs=Cancel Scheduled Attachment Deletion From Local FS +cancel.src.upload.service=Cancel SRC Upload Service cannot.add.attachments.before.saving.the.document=Cannot add attachments before saving the document cannot.continue.to.merge.of.releases=Cannot continue to merge of releases: can.not.get.products=Can not get products! @@ -1310,6 +1311,7 @@ schedule=Schedule schedule.cve.service=Schedule CVE service schedule.attachment.deletion.from.local.fs=Schedule Attachment Deletion From Local FS schedule.offset=Schedule Offset +schedule.src.upload.service=Schedule SRC Upload Service schedule.task.administration=Schedule Task Administration scope=Scope screenshot.of.website=Screenshot of Website @@ -1398,6 +1400,7 @@ split=Split split.data=2. Split data split.data.from.current.component.to.target.component=Split data from current component to target component split.into=Split into +src.upload = SRC Upload stakeholder=Stakeholder standalone=Standalone start=Start diff --git a/frontend/sw360-portlet/src/main/resources/content/Language_ja.properties b/frontend/sw360-portlet/src/main/resources/content/Language_ja.properties index adbfbc810a..b0e0ab112b 100644 --- a/frontend/sw360-portlet/src/main/resources/content/Language_ja.properties +++ b/frontend/sw360-portlet/src/main/resources/content/Language_ja.properties @@ -1395,6 +1395,7 @@ split=分離 split.data=2. データ分離 split.data.from.current.component.to.target.component=現在のコンポーネントから対象のコンポーネントにデータを分離 split.into=分離 +src.upload=SRC Upload stakeholder=ステークホルダー standalone=単体 start=開始 diff --git a/frontend/sw360-portlet/src/main/resources/content/Language_vi.properties b/frontend/sw360-portlet/src/main/resources/content/Language_vi.properties index 35a089c2b4..6f3fcb4f4e 100644 --- a/frontend/sw360-portlet/src/main/resources/content/Language_vi.properties +++ b/frontend/sw360-portlet/src/main/resources/content/Language_vi.properties @@ -1392,6 +1392,7 @@ split=Split split.data=2. Split data split.data.from.current.component.to.target.component=Split data from current component to target component split.into=Split into +src.upload=SRC Upload stakeholder=Các bên liên quan standalone=Độc lập start=Bắt đầu diff --git a/frontend/sw360-portlet/src/main/resources/content/Language_zh.properties b/frontend/sw360-portlet/src/main/resources/content/Language_zh.properties index 50ad175640..30f7f4227c 100644 --- a/frontend/sw360-portlet/src/main/resources/content/Language_zh.properties +++ b/frontend/sw360-portlet/src/main/resources/content/Language_zh.properties @@ -1385,6 +1385,7 @@ split=拆分 split.data=2.拆分数据 split.data.from.current.component.to.target.component=将数据从当前组件拆分到目标组件 split.into=分割成 +src.upload=SRC Upload stakeholder=利益相关者 standalone=独立 start=开始 diff --git a/libraries/commonIO/src/main/java/org/eclipse/sw360/commonIO/AttachmentFrontendUtils.java b/libraries/commonIO/src/main/java/org/eclipse/sw360/commonIO/AttachmentFrontendUtils.java index 0c5361f8f0..c6c272b740 100644 --- a/libraries/commonIO/src/main/java/org/eclipse/sw360/commonIO/AttachmentFrontendUtils.java +++ b/libraries/commonIO/src/main/java/org/eclipse/sw360/commonIO/AttachmentFrontendUtils.java @@ -121,7 +121,11 @@ public Attachment uploadAttachmentContent(AttachmentContent attachmentContent, I if (attachmentContent != null) { try { attachmentStreamConnector.uploadAttachment(attachmentContent, fileStream); - return CommonUtils.getNewAttachment(sw360User, attachmentContent.getId(), attachmentContent.getFilename()); + if (sw360User != null) { + return CommonUtils.getNewAttachment(sw360User, attachmentContent.getId(), attachmentContent.getFilename()); + } else { + return CommonUtils.getNewAttachment(attachmentContent.getId(), attachmentContent.getFilename()); + } } catch (TException e) { log.error("Error saving attachment part", e); } diff --git a/libraries/datahandler/src/main/java/org/eclipse/sw360/datahandler/common/CommonUtils.java b/libraries/datahandler/src/main/java/org/eclipse/sw360/datahandler/common/CommonUtils.java index 33d472f806..74d3f38016 100644 --- a/libraries/datahandler/src/main/java/org/eclipse/sw360/datahandler/common/CommonUtils.java +++ b/libraries/datahandler/src/main/java/org/eclipse/sw360/datahandler/common/CommonUtils.java @@ -330,17 +330,31 @@ public static Optional getAttachmentOptional(final String attachment @NotNull public static Attachment getNewAttachment(User user, String attachmentContentId, String fileName) { + return initializeAttachmentCommonFields(user, attachmentContentId, fileName); + } + + @NotNull + public static Attachment getNewAttachment(String attachmentContentId, String fileName) { + return initializeAttachmentCommonFields(null, attachmentContentId, fileName); + } + + @NotNull + private static Attachment initializeAttachmentCommonFields(User user, String attachmentContentId, String fileName) { Attachment attachment = new Attachment(); - attachment.setCreatedBy(user.getEmail()); attachment.setCreatedOn(SW360Utils.getCreatedOn()); attachment.setCreatedComment(""); - attachment.setCreatedTeam(user.getDepartment()); attachment.setFilename(fileName); attachment.setAttachmentContentId(attachmentContentId); attachment.setAttachmentType(AttachmentType.DOCUMENT); attachment.setCheckStatus(CheckStatus.NOTCHECKED); attachment.setCheckedComment(""); attachment.setSha1(""); + + if (user != null) { + attachment.setCreatedBy(user.getEmail()); + attachment.setCreatedTeam(user.getDepartment()); + } + return attachment; } diff --git a/libraries/datahandler/src/main/java/org/eclipse/sw360/datahandler/common/SW360Constants.java b/libraries/datahandler/src/main/java/org/eclipse/sw360/datahandler/common/SW360Constants.java index ea05e1b3c8..6758a26879 100644 --- a/libraries/datahandler/src/main/java/org/eclipse/sw360/datahandler/common/SW360Constants.java +++ b/libraries/datahandler/src/main/java/org/eclipse/sw360/datahandler/common/SW360Constants.java @@ -116,6 +116,9 @@ public class SW360Constants { public static final String TOOL_VENDOR; public static final UserGroup SBOM_IMPORT_EXPORT_ACCESS_USER_ROLE; public static final boolean ENABLE_FLEXIBLE_PROJECT_RELEASE_RELATIONSHIP; + public static final String URL_FORMATS; + public static final String SRC_ATTACHMENT_UPLOADER_EMAIL; + public static final String SRC_ATTACHMENT_DOWNLOAD_LOCATION; /** * Hashmap containing the name field for each type. @@ -217,6 +220,9 @@ private SW360Constants() { SBOM_IMPORT_EXPORT_ACCESS_USER_ROLE = UserGroup.valueOf(props.getProperty("sbom.import.export.access.usergroup", UserGroup.USER.name())); ENABLE_FLEXIBLE_PROJECT_RELEASE_RELATIONSHIP = Boolean.parseBoolean( System.getProperty("RunTestFlexibleRelationship", props.getProperty("enable.flexible.project.release.relationship", "false"))); + URL_FORMATS = props.getProperty("source.download.formats",""); + SRC_ATTACHMENT_UPLOADER_EMAIL = props.getProperty("source.code.attachment.uploader.email", ""); + SRC_ATTACHMENT_DOWNLOAD_LOCATION = props.getProperty("src.attachment.download.location", ""); } private static Map.Entry pair(TFieldIdEnum field, String displayName){ diff --git a/libraries/datahandler/src/main/java/org/eclipse/sw360/datahandler/thrift/ThriftClients.java b/libraries/datahandler/src/main/java/org/eclipse/sw360/datahandler/thrift/ThriftClients.java index b33803bb50..efa5214cc5 100644 --- a/libraries/datahandler/src/main/java/org/eclipse/sw360/datahandler/thrift/ThriftClients.java +++ b/libraries/datahandler/src/main/java/org/eclipse/sw360/datahandler/thrift/ThriftClients.java @@ -105,6 +105,7 @@ public class ThriftClients { public static final String SVM_TRACKING_FEEDBACK_SERVICE = "svmTrackingFeedbackService"; public static final String DELETE_ATTACHMENT_SERVICE = "deleteattachmentService"; public static final String IMPORT_DEPARTMENT_SERVICE = "importdepartmentService"; + public static final String SRC_UPLOAD_SERVICE = "srcAttachmentUploadService"; static { diff --git a/libraries/datahandler/src/main/thrift/components.thrift b/libraries/datahandler/src/main/thrift/components.thrift index c858e67ec3..b71cf56c9d 100644 --- a/libraries/datahandler/src/main/thrift/components.thrift +++ b/libraries/datahandler/src/main/thrift/components.thrift @@ -946,6 +946,11 @@ service ComponentService { **/ RequestStatus updateReleasesWithSvmTrackingFeedback(); + /** + * uploads source code attachment to releases + **/ + RequestStatus uploadSourceCodeAttachmentToReleases(); + /** * Method to ensure uniqueness of identifiers, used by database sanitation portlet, * return map of name to ids