Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: Restructure packages, add workflow run sync + webhook handler #53

Merged
merged 9 commits into from
Dec 2, 2024
1 change: 1 addition & 0 deletions .env.example
Original file line number Diff line number Diff line change
Expand Up @@ -11,3 +11,4 @@ WEBHOOK_SECRET=123
REPOSITORY_NAME=
ORGANIZATION_NAME=
GITHUB_AUTH_TOKEN=
RUN_ON_STARTUP_COOLDOWN=0
1 change: 0 additions & 1 deletion compose.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -58,7 +58,6 @@ services:
- "80:4200"

application-server:
image: ghcr.io/ls1intum/helios/application-server:latest
build:
context: server/application-server
ports:
Expand Down
3 changes: 2 additions & 1 deletion server/application-server/.env.example
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,8 @@ DATASOURCE_URL=jdbc:postgresql://127.0.0.1/helios
DATASOURCE_USERNAME=helios
DATASOURCE_PASSWORD=helios
NATS_SERVER=localhost:4222
NATS_AUTH_TOKEN='5760e8ae09adfb2756f9f8cd5cb2caa704cd3f549eaa9298be843ceb165185d815b81f90c680fa7f626b7cd63abf6ac9'
NATS_AUTH_TOKEN=5760e8ae09adfb2756f9f8cd5cb2caa704cd3f549eaa9298be843ceb165185d815b81f90c680fa7f626b7cd63abf6ac9
REPOSITORY_NAME=<repository_name e.g. ls1intum/Helios>
ORGANIZATION_NAME=<organization_name e.g. ls1intum>
GITHUB_AUTH_TOKEN=
RUN_ON_STARTUP_COOLDOWN=0 <when server starts, it first checks the latest run of sync, if it is less than this value in minutes, it will not run the sync again>
Original file line number Diff line number Diff line change
@@ -1,19 +1,11 @@
package de.tum.cit.aet.helios;
import java.util.Map;
import java.util.stream.Collectors;

import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springdoc.core.customizers.OpenApiCustomizer;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;

import io.swagger.v3.oas.annotations.OpenAPIDefinition;
import io.swagger.v3.oas.annotations.info.Contact;
import io.swagger.v3.oas.annotations.info.Info;
import io.swagger.v3.oas.annotations.info.License;
import io.swagger.v3.oas.annotations.servers.Server;
import io.swagger.v3.oas.models.media.Schema;

@Configuration
@OpenAPIDefinition(
Expand Down
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
package de.tum.cit.aet.helios.gitprovider.common.github;
package de.tum.cit.aet.helios.github;

import jakarta.persistence.Id;
import jakarta.persistence.MappedSuperclass;
Expand Down
Original file line number Diff line number Diff line change
@@ -1,20 +1,20 @@
package de.tum.cit.aet.helios.gitprovider.common.github;
package de.tum.cit.aet.helios.github;

import org.kohsuke.github.GHObject;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.core.convert.converter.Converter;
import org.springframework.data.convert.ReadingConverter;
import org.springframework.lang.NonNull;

import de.tum.cit.aet.helios.util.DateUtil;
import lombok.extern.log4j.Log4j2;

import java.io.IOException;

@ReadingConverter
@Log4j2
public abstract class BaseGitServiceEntityConverter<S extends GHObject, T extends BaseGitServiceEntity>
implements Converter<S, T> {

private static final Logger logger = LoggerFactory.getLogger(BaseGitServiceEntityConverter.class);

abstract public T update(@NonNull S source, @NonNull T target);

protected void convertBaseFields(S source, T target) {
Expand All @@ -28,14 +28,14 @@ protected void convertBaseFields(S source, T target) {
try {
target.setCreatedAt(DateUtil.convertToOffsetDateTime(source.getCreatedAt()));
} catch (IOException e) {
logger.error("Failed to convert createdAt field for source {}: {}", source.getId(), e.getMessage());
log.error("Failed to convert createdAt field for source {}: {}", source.getId(), e.getMessage());
target.setCreatedAt(null);
}

try {
target.setUpdatedAt(DateUtil.convertToOffsetDateTime(source.getUpdatedAt()));
} catch (IOException e) {
logger.error("Failed to convert updatedAt field for source {}: {}", source.getId(), e.getMessage());
log.error("Failed to convert updatedAt field for source {}: {}", source.getId(), e.getMessage());
target.setUpdatedAt(null);
}
}
Expand Down
Original file line number Diff line number Diff line change
@@ -1,15 +1,13 @@
package de.tum.cit.aet.helios.config;
package de.tum.cit.aet.helios.github;

import lombok.Getter;
import lombok.extern.log4j.Log4j2;
import okhttp3.Cache;
import okhttp3.OkHttpClient;
import okhttp3.logging.HttpLoggingInterceptor;
import org.kohsuke.github.GitHub;
import org.kohsuke.github.GitHubBuilder;
import org.kohsuke.github.extras.okhttp3.OkHttpGitHubConnector;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
Expand All @@ -20,10 +18,8 @@
import java.util.concurrent.TimeUnit;

@Configuration
@Log4j2
public class GitHubConfig {

private static final Logger logger = LoggerFactory.getLogger(GitHubConfig.class);

@Getter
@Value("${github.organizationName}")
private String organizationName;
Expand All @@ -42,25 +38,24 @@ public class GitHubConfig {

private final Environment environment;

@Autowired
public GitHubConfig(Environment environment) {
this.environment = environment;
}

@Bean
public GitHub createGitHubClientWithCache() {
if (ghAuthToken == null || ghAuthToken.isEmpty()) {
logger.error("GitHub auth token is not provided! GitHub client will be disabled.");
log.error("GitHub auth token is not provided! GitHub client will be disabled.");
return GitHub.offline();
}

// Set up a logging interceptor for debugging
HttpLoggingInterceptor loggingInterceptor = new HttpLoggingInterceptor();
if (environment.matchesProfiles("debug")) {
logger.warn("The requests to GitHub will be logged with the full body. This exposes sensitive data such as OAuth tokens. Use only for debugging!");
log.warn("The requests to GitHub will be logged with the full body. This exposes sensitive data such as OAuth tokens. Use only for debugging!");
loggingInterceptor.setLevel(HttpLoggingInterceptor.Level.BODY);
} else {
logger.info("The requests to GitHub will be logged with the basic information.");
log.info("The requests to GitHub will be logged with the basic information.");
loggingInterceptor.setLevel(HttpLoggingInterceptor.Level.BASIC);
}

Expand All @@ -70,9 +65,9 @@ public GitHub createGitHubClientWithCache() {
File cacheDir = new File("./build/github-cache");
Cache cache = new Cache(cacheDir, cacheSize * 1024L * 1024L);
builder.cache(cache);
logger.info("Cache is enabled with TTL {} seconds and size {} MB", cacheTtl, cacheSize);
log.info("Cache is enabled with TTL {} seconds and size {} MB", cacheTtl, cacheSize);
} else {
logger.info("Cache is disabled");
log.info("Cache is disabled");
}

// Configure OkHttpClient with the cache and logging
Expand All @@ -90,16 +85,16 @@ public GitHub createGitHubClientWithCache() {
.withOAuthToken(ghAuthToken)
.build();
if (!github.isCredentialValid()) {
logger.error("Invalid GitHub credentials!");
log.error("Invalid GitHub credentials!");
throw new IllegalStateException("Invalid GitHub credentials");
}
logger.info("GitHub client initialized successfully");
log.info("GitHub client initialized successfully");
return github;
} catch (IOException e) {
logger.error("Failed to initialize GitHub client: {}", e.getMessage());
log.error("Failed to initialize GitHub client: {}", e.getMessage());
throw new RuntimeException("GitHub client initialization failed", e);
} catch (Exception e) {
logger.error("An unexpected error occurred during GitHub client initialization: {}", e.getMessage());
log.error("An unexpected error occurred during GitHub client initialization: {}", e.getMessage());
throw new RuntimeException("Unexpected error during GitHub client initialization", e);
}
}
Expand Down
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
package de.tum.cit.aet.helios.gitprovider.common.github;
package de.tum.cit.aet.helios.github;

import java.io.IOException;
import java.io.StringReader;
Expand All @@ -7,18 +7,16 @@
import org.kohsuke.github.GHEvent;
import org.kohsuke.github.GHEventPayload;
import org.kohsuke.github.GitHub;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import io.nats.client.Message;
import io.nats.client.MessageHandler;
import lombok.extern.log4j.Log4j2;

import org.springframework.stereotype.Component;

@Component
@Log4j2
public abstract class GitHubMessageHandler<T extends GHEventPayload> implements MessageHandler {

private static final Logger logger = LoggerFactory.getLogger(GitHubMessageHandler.class);

private final Class<T> payloadType;

protected GitHubMessageHandler(Class<T> payloadType) {
Expand All @@ -30,7 +28,7 @@ public void onMessage(Message msg) {
String eventType = getHandlerEvent().name().toLowerCase();
String subject = msg.getSubject();
if (!subject.endsWith(eventType)) {
logger.error("Received message on unexpected subject: {}, expected to end with {}", subject, eventType);
log.error("Received message on unexpected subject: {}, expected to end with {}", subject, eventType);
return;
}

Expand All @@ -40,9 +38,9 @@ public void onMessage(Message msg) {
T eventPayload = GitHub.offline().parseEventPayload(reader, payloadType);
handleEvent(eventPayload);
} catch (IOException e) {
logger.error("Failed to parse payload for subject {}: {}", subject, e.getMessage(), e);
log.error("Failed to parse payload for subject {}: {}", subject, e.getMessage(), e);
} catch (Exception e) {
logger.error("Unexpected error while handling message for subject {}: {}", subject, e.getMessage(), e);
log.error("Unexpected error while handling message for subject {}: {}", subject, e.getMessage(), e);
}
}

Expand Down
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
package de.tum.cit.aet.helios.gitprovider.common.github;
package de.tum.cit.aet.helios.github;

import java.util.ArrayList;
import java.util.HashMap;
Expand Down
Original file line number Diff line number Diff line change
@@ -1,31 +1,26 @@
package de.tum.cit.aet.helios.gitprovider;
package de.tum.cit.aet.helios.github;

import de.tum.cit.aet.helios.config.GitHubConfig;
import org.kohsuke.github.GHOrganization;
import org.kohsuke.github.GHRepository;
import org.kohsuke.github.GHWorkflow;
import org.kohsuke.github.GitHub;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;

import lombok.extern.log4j.Log4j2;

import java.io.IOException;
import java.util.List;
import java.util.Map;

@Service
@Log4j2
public class GitHubService {

private static final Logger logger = LoggerFactory.getLogger(GitHubService.class);

private final GitHub github;

private final GitHubConfig gitHubConfig;

private GHOrganization gitHubOrganization;

@Autowired
public GitHubService(GitHub github, GitHubConfig gitHubConfig) {
this.github = github;
this.gitHubConfig = gitHubConfig;
Expand All @@ -41,7 +36,7 @@ public GHOrganization getOrganizationClient() throws IOException {
if (gitHubOrganization == null) {
final String organizationName = gitHubConfig.getOrganizationName();
if (organizationName == null || organizationName.isEmpty()) {
logger.error("No organization name provided in the configuration. GitHub organization client will not be initialized.");
log.error("No organization name provided in the configuration. GitHub organization client will not be initialized.");
throw new RuntimeException("No organization name provided in the configuration.");
}
gitHubOrganization = github.getOrganization(organizationName);
Expand Down
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
package de.tum.cit.aet.helios.syncing;
package de.tum.cit.aet.helios.github.sync;

import jakarta.persistence.*;
import lombok.Getter;
Expand Down
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
package de.tum.cit.aet.helios.syncing;
package de.tum.cit.aet.helios.github.sync;

import org.springframework.data.jpa.repository.JpaRepository;
import org.springframework.stereotype.Repository;
Expand Down
Original file line number Diff line number Diff line change
@@ -1,19 +1,18 @@
package de.tum.cit.aet.helios.syncing;
package de.tum.cit.aet.helios.github.sync;

import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.boot.context.event.ApplicationReadyEvent;
import org.springframework.context.event.EventListener;
import org.springframework.core.annotation.Order;
import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Component;

import lombok.extern.log4j.Log4j2;

@Order(value = 2)
@Component
@Log4j2
public class GitHubDataSyncScheduler {

private static final Logger logger = LoggerFactory.getLogger(GitHubDataSyncScheduler.class);
private final GitHubDataSyncService dataSyncService;

@Value("${monitoring.runOnStartup:true}")
Expand All @@ -26,16 +25,16 @@ public GitHubDataSyncScheduler(GitHubDataSyncService dataSyncService) {
@EventListener(ApplicationReadyEvent.class)
public void run() {
if (runOnStartup) {
logger.info("Starting initial GitHub data sync...");
log.info("Starting initial GitHub data sync...");
dataSyncService.syncData();
logger.info("Initial GitHub data sync completed.");
log.info("Initial GitHub data sync completed.");
}
}

@Scheduled(cron = "${monitoring.repository-sync-cron}")
public void syncDataCron() {
logger.info("Starting scheduled GitHub data sync...");
log.info("Starting scheduled GitHub data sync...");
dataSyncService.syncData();
logger.info("Scheduled GitHub data sync completed.");
log.info("Scheduled GitHub data sync completed.");
}
}
Loading