Merge pull request #264 from intelops/ttl2

added TTL
intelops · Nov 2, 2023 · a24e7d7 · a24e7d7
2 parents 7d58985 + f74a745
commit a24e7d7
Show file tree

Hide file tree

Showing 20 changed files with 108 additions and 40 deletions.
diff --git a/sql/20230915101223_events.up.sql b/sql/20230915101223_events.up.sql
@@ -11,5 +11,8 @@ CREATE TABLE IF NOT EXISTS events (
 	Host        String,
 	Event       String,
 	FirstTime   String,
-	LastTime    String
-) engine=File(TabSeparated);
+	LastTime    String,
+	ExpiryDate DateTime DEFAULT now() + INTERVAL 6 MONTH
+) ENGINE = MergeTree() 
+ORDER BY ExpiryDate 
+TTL ExpiryDate;
diff --git a/sql/20230915101317_rakkess.up.sql b/sql/20230915101317_rakkess.up.sql
@@ -5,5 +5,9 @@ CREATE TABLE IF NOT EXISTS rakkess (
 	Delete      String,
 	List        String,
 	Update      String,
-	EventTime   DateTime('UTC')
-) engine=File(TabSeparated);
+	EventTime   DateTime('UTC'),
+	ExpiryDate DateTime DEFAULT now() + INTERVAL 6 MONTH
+) ENGINE = MergeTree() 
+ORDER BY ExpiryDate 
+TTL ExpiryDate;
+
diff --git a/sql/20230915101358_DeprecatedAPIs.up.sql b/sql/20230915101358_DeprecatedAPIs.up.sql
@@ -5,5 +5,9 @@ CREATE TABLE IF NOT EXISTS DeprecatedAPIs (
 	Kind            String,
 	Deprecated      UInt8,
 	Scope           String,
-	EventTime       DateTime('UTC')
-) engine=File(TabSeparated);
+	EventTime       DateTime('UTC'),
+	ExpiryDate DateTime DEFAULT now() + INTERVAL 6 MONTH
+) ENGINE = MergeTree() 
+ORDER BY ExpiryDate 
+TTL ExpiryDate;
+
diff --git a/sql/20230915101437_DeletedAPIs.up.sql b/sql/20230915101437_DeletedAPIs.up.sql
@@ -7,5 +7,8 @@ CREATE TABLE IF NOT EXISTS DeletedAPIs (
 	Name            String,
 	Deleted         UInt8,
 	Scope           String,
-	EventTime       DateTime('UTC')
-) engine=File(TabSeparated);
+	EventTime       DateTime('UTC'),
+	ExpiryDate DateTime DEFAULT now() + INTERVAL 6 MONTH
+) ENGINE = MergeTree() 
+ORDER BY ExpiryDate 
+TTL ExpiryDate;
diff --git a/sql/20230915101512_jfrogcontainerpush.up.sql b/sql/20230915101512_jfrogcontainerpush.up.sql
@@ -8,5 +8,9 @@ CREATE TABLE IF NOT EXISTS jfrogcontainerpush (
 	ImageName      String,
 	Tag            String,
 	Event          String,
-    EventTime      DateTime('UTC')
-) engine=File(TabSeparated);
+    EventTime      DateTime('UTC'),
+	ExpiryDate DateTime DEFAULT now() + INTERVAL 6 MONTH
+) ENGINE = MergeTree() 
+ORDER BY ExpiryDate 
+TTL ExpiryDate;
+
diff --git a/sql/20230915101549_getall_resources.up.sql b/sql/20230915101549_getall_resources.up.sql
@@ -4,5 +4,9 @@ CREATE TABLE IF NOT EXISTS getall_resources (
 	Kind        String,
 	Resource    String,
 	Age         String,
-    EventTime   DateTime('UTC')
-) engine=File(TabSeparated);
+    EventTime   DateTime('UTC'),
+	ExpiryDate DateTime DEFAULT now() + INTERVAL 6 MONTH
+) ENGINE = MergeTree() 
+ORDER BY ExpiryDate 
+TTL ExpiryDate;
+
diff --git a/sql/20230915101643_outdated_images.up.sql b/sql/20230915101643_outdated_images.up.sql
@@ -6,5 +6,9 @@ CREATE TABLE IF NOT EXISTS outdated_images (
 	CurrentTag      String,
 	LatestVersion   String,
 	VersionsBehind  Int64,
-    EventTime       DateTime('UTC')
-) engine=File(TabSeparated);
+    EventTime       DateTime('UTC'),
+	ExpiryDate DateTime DEFAULT now() + INTERVAL 6 MONTH
+) ENGINE = MergeTree() 
+ORDER BY ExpiryDate 
+TTL ExpiryDate;
+
diff --git a/sql/20230915101736_kubescore.up.sql b/sql/20230915101736_kubescore.up.sql
@@ -3,5 +3,9 @@ CREATE TABLE IF NOT EXISTS kubescore (
 	namespace       String,
 	cluster_name    String,
 	recommendations String,
-    EventTime       DateTime('UTC')
-) engine=File(TabSeparated);
+    EventTime       DateTime('UTC'),
+	ExpiryDate DateTime DEFAULT now() + INTERVAL 6 MONTH
+) ENGINE = MergeTree() 
+ORDER BY ExpiryDate 
+TTL ExpiryDate;
+
diff --git a/sql/20230915101811_trivy_vul.up.sql b/sql/20230915101811_trivy_vul.up.sql
@@ -14,5 +14,9 @@ CREATE TABLE IF NOT EXISTS trivy_vul (
 	vul_title             String,
 	vul_severity          String,
 	vul_published_date    DateTime('UTC'),
-	vul_last_modified_date DateTime('UTC')
-) engine=File(TabSeparated);
+	vul_last_modified_date DateTime('UTC'),
+	ExpiryDate DateTime DEFAULT now() + INTERVAL 6 MONTH
+) ENGINE = MergeTree() 
+ORDER BY ExpiryDate 
+TTL ExpiryDate;
+
diff --git a/sql/20230915101844_trivy_misconfig.up.sql b/sql/20230915101844_trivy_misconfig.up.sql
@@ -14,5 +14,9 @@ CREATE TABLE IF NOT EXISTS trivy_misconfig (
 	misconfig_resolution String,
 	misconfig_severity String,
 	misconfig_status   String,
-    EventTime          DateTime('UTC')
-) engine=File(TabSeparated);
+    EventTime          DateTime('UTC'),
+	ExpiryDate DateTime DEFAULT now() + INTERVAL 6 MONTH
+) ENGINE = MergeTree() 
+ORDER BY ExpiryDate 
+TTL ExpiryDate;
+
diff --git a/sql/20230915101910_trivyimage.up.sql b/sql/20230915101910_trivyimage.up.sql
@@ -10,5 +10,8 @@ CREATE TABLE IF NOT EXISTS trivyimage (
 	vul_title           String,
 	vul_severity        String,
 	vul_published_date  DateTime('UTC'),
-	vul_last_modified_date DateTime('UTC')
-) engine=File(TabSeparated);
+	vul_last_modified_date DateTime('UTC'),
+	ExpiryDate DateTime DEFAULT now() + INTERVAL 6 MONTH
+) ENGINE = MergeTree() 
+ORDER BY ExpiryDate 
+TTL ExpiryDate;
diff --git a/sql/20230915102122_dockerhubbuild.up.sql b/sql/20230915102122_dockerhubbuild.up.sql
@@ -5,5 +5,8 @@ CREATE TABLE IF NOT EXISTS dockerhubbuild (
 	DateCreated   String,
 	Owner         String,
 	Event         String,
-    EventTime     DateTime('UTC')
-) engine=File(TabSeparated);
+    EventTime     DateTime('UTC'),
+	ExpiryDate DateTime DEFAULT now() + INTERVAL 6 MONTH
+) ENGINE = MergeTree() 
+ORDER BY ExpiryDate 
+TTL ExpiryDate;
diff --git a/sql/20230915102157_azurecontainerpush.up.sql b/sql/20230915102157_azurecontainerpush.up.sql
@@ -6,5 +6,8 @@ CREATE TABLE IF NOT EXISTS azurecontainerpush (
 	Event          String,
 	Size           Int32,
 	SHAID          String,
-    EventTime DateTime('UTC')
-) engine=File(TabSeparated);
+    EventTime DateTime('UTC'),
+	ExpiryDate DateTime DEFAULT now() + INTERVAL 6 MONTH
+) ENGINE = MergeTree() 
+ORDER BY ExpiryDate 
+TTL ExpiryDate;
diff --git a/sql/20230915102229_quaycontainerpush.up.sql b/sql/20230915102229_quaycontainerpush.up.sql
@@ -6,5 +6,8 @@ CREATE TABLE IF NOT EXISTS quaycontainerpush (
 	homePage      String,
 	tag           String,
 	Event         String,
-    EventTime DateTime('UTC')
-) engine=File(TabSeparated);
+    EventTime DateTime('UTC'),
+	ExpiryDate DateTime DEFAULT now() + INTERVAL 6 MONTH
+) ENGINE = MergeTree() 
+ORDER BY ExpiryDate 
+TTL ExpiryDate;
diff --git a/sql/20230915102314_trivysbom.up.sql b/sql/20230915102314_trivysbom.up.sql
@@ -19,5 +19,8 @@ CREATE TABLE IF NOT EXISTS trivysbom (
 	component_hash_content String,
 	component_license_exp String,
 	component_purl        String,
-	dependency_ref        String
-) engine=File(TabSeparated);
+	dependency_ref        String,
+	ExpiryDate DateTime DEFAULT now() + INTERVAL 6 MONTH
+) ENGINE = MergeTree() 
+ORDER BY ExpiryDate 
+TTL ExpiryDate;
diff --git a/sql/20230915102348_azure_devops.up.sql b/sql/20230915102348_azure_devops.up.sql
@@ -6,5 +6,8 @@ CREATE TABLE IF NOT EXISTS azure_devops (
 	EventType    String,
 	RepoName     String,
 	TimeStamp    DateTime('UTC'),
-	Event        String
-) engine=File(TabSeparated);
+	Event        String,
+	ExpiryDate DateTime DEFAULT now() + INTERVAL 6 MONTH
+) ENGINE = MergeTree() 
+ORDER BY ExpiryDate 
+TTL ExpiryDate;
diff --git a/sql/20230915102437_github.up.sql b/sql/20230915102437_github.up.sql
@@ -6,5 +6,8 @@ CREATE TABLE IF NOT EXISTS github (
 	EventType    String,
 	RepoName     String,
 	TimeStamp    DateTime('UTC'),
-	Event        String
-) engine=File(TabSeparated);
+	Event        String,
+	ExpiryDate DateTime DEFAULT now() + INTERVAL 6 MONTH
+) ENGINE = MergeTree() 
+ORDER BY ExpiryDate 
+TTL ExpiryDate;
diff --git a/sql/20230915102739_gitlab.up.sql b/sql/20230915102739_gitlab.up.sql
@@ -6,5 +6,8 @@ CREATE TABLE IF NOT EXISTS gitlab (
 	EventType    String,
 	RepoName     String,
 	TimeStamp    DateTime('UTC'),
-	Event        String
-) engine=File(TabSeparated);
+	Event        String,
+	ExpiryDate DateTime DEFAULT now() + INTERVAL 6 MONTH
+) ENGINE = MergeTree() 
+ORDER BY ExpiryDate 
+TTL ExpiryDate;
diff --git a/sql/20230915102817_bitbucket.up.sql b/sql/20230915102817_bitbucket.up.sql
@@ -6,5 +6,8 @@ CREATE TABLE IF NOT EXISTS bitbucket (
 	EventType    String,
 	RepoName     String,
 	TimeStamp    DateTime('UTC'),
-	Event        String
-) engine=File(TabSeparated);
+	Event        String,
+	ExpiryDate DateTime DEFAULT now() + INTERVAL 6 MONTH
+) ENGINE = MergeTree() 
+ORDER BY ExpiryDate 
+TTL ExpiryDate;
diff --git a/sql/20230915102843_gitea.up.sql b/sql/20230915102843_gitea.up.sql
@@ -6,5 +6,8 @@ CREATE TABLE IF NOT EXISTS gitea (
 	EventType    String,
 	RepoName     String,
 	TimeStamp    DateTime('UTC'),
-	Event        String
-) engine=File(TabSeparated);
+	Event        String,
+	ExpiryDate DateTime DEFAULT now() + INTERVAL 6 MONTH
+) ENGINE = MergeTree() 
+ORDER BY ExpiryDate 
+TTL ExpiryDate;