Skip to content

Commit 8d3728f

Browse files
okynosokynos
committed
Added database and scanner integration first performance tests
1 parent 749cde5 commit 8d3728f

File tree

9 files changed

+177
-80
lines changed

9 files changed

+177
-80
lines changed

‎Cargo.toml‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@ itertools = "0.13.0"
1414
hex = "0.4.3"
1515
futures = "0.3.21"
1616
sha3 = { version = "0.10.0", default-features = false }
17+
sha2 = { version = "0.10.8" }
1718
log = { version = "0.4.11", default-features = false }
1819
gethostname = { version = "0.5.0", default-features = false }
1920
uuid = { version = "1.0.0", default-features = false, features = ["v4"] }
@@ -25,6 +26,7 @@ time = { version = "0.3.17", default-features = false }
2526
ctrlc = { version = "3.3.1", default-features = false, features = ["termination"] }
2627
log-panics = { version = "2.1.0", features = ["with-backtrace"]}
2728
rusqlite = { version = "0.32.1", features = ["bundled"]}
29+
walkdir = "2.5.0"
2830

2931
[dependencies.regex]
3032
version = "1.3"

‎src/auditevent.rs‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@ use serde_json::{json, to_string};
99
use reqwest::Client;
1010
use std::collections::HashMap;
1111
use std::path::PathBuf;
12+
use sha3::{Digest, Sha3_512};
1213

1314
use crate::appconfig;
1415
use crate::appconfig::*;
@@ -145,7 +146,7 @@ impl Event {
145146
size: utils::get_file_size(path["name"].clone().as_str()),
146147
checksum: hash::get_checksum(format!("{}/{}",
147148
parent["name"].clone(), path["name"].clone()),
148-
cfg.events_max_file_checksum),
149+
cfg.events_max_file_checksum, Sha3_512::new()),
149150
fpid: utils::get_pid(),
150151
system: String::from(utils::get_os()),
151152

‎src/db.rs‎

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@ pub struct DBFile {
1212
pub timestamp: String,
1313
pub hash: String,
1414
pub path: String,
15-
pub size: u32
15+
pub size: u64
1616
}
1717

1818
pub struct DB {
@@ -69,8 +69,8 @@ impl DB {
6969
(),
7070
);
7171
match result {
72-
Ok(_v) => println!("GOOD"),
73-
Err(e) => println!("ERROR: {:?}", e)
72+
Ok(_v) => info!("Database successfully created."),
73+
Err(e) => error!("Error creating database, Error: '{}'", e)
7474
}
7575
self.close(connection);
7676
}
@@ -104,7 +104,7 @@ impl DB {
104104
path: row.get(3).unwrap(),
105105
size: row.get(4).unwrap()
106106
})
107-
);
107+
).unwrap();
108108

109109
self.close(connection);
110110
data
@@ -137,6 +137,7 @@ impl DB {
137137
impl fmt::Debug for DBFile {
138138
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result{
139139
f.debug_tuple("")
140+
.field(&self.id)
140141
.field(&self.timestamp)
141142
.field(&self.hash)
142143
.field(&self.path)

‎src/hash.rs‎

Lines changed: 83 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@ const READ_CAPACITY: usize = 1024 * 1024 * 8; // Read file in chunks of 8MB
55

66
// To get file checksums
77
use hex::{encode, decode};
8-
use sha3::{Sha3_512, Digest};
8+
use sha3::Digest;
99
// To log the program process
1010
use log::*;
1111
// To manage hex to ascii conversion
@@ -16,44 +16,103 @@ use std::path::Path;
1616
// To read file content
1717
use std::io::{BufRead, BufReader};
1818

19-
// To calculate file content hash in sha512 format (SHA3 implementation)
20-
pub fn get_checksum(filename: String, read_limit: usize) -> String {
21-
let mut hasher = Sha3_512::new();
19+
// To calculate file content hash (SHA3 implementation)
20+
pub fn get_checksum<T: Digest>(filename: String, read_limit: usize, mut hasher: T) -> String {
21+
//let mut hasher = Sha3_512::new();
2222
let mut length = 1;
2323
let mut iteration = 0;
2424
let mut data_read = 0;
25+
let limit: u64 = (read_limit * 1024 * 1024).try_into().unwrap();
2526

2627
if Path::new(&filename).is_file() {
2728
debug!("Getting hash of file: {}", filename);
2829
match File::open(filename.clone()){
2930
Ok(file) => {
31+
let size = file.metadata().unwrap().len();
3032
let mut reader = BufReader::with_capacity(READ_CAPACITY, file);
3133

32-
while length > 0 && data_read <= read_limit {
33-
if iteration == 2 {
34-
debug!("Big file detected, the hash will take a while");
35-
}
36-
37-
length = {
38-
match reader.fill_buf(){
39-
Ok(buffer) =>{
40-
hasher.update(buffer);
41-
buffer.len()
42-
},
43-
Err(e) => {
44-
debug!("Cannot read file. Checksum set to 'UNKNOWN', error: {}", e);
45-
0
46-
}
34+
35+
if size > limit {
36+
info!("File '{}' checksum skipped. File size is above limit.", filename);
37+
String::from("UNKNOWN")
38+
}else{
39+
while length > 0 && data_read <= read_limit {
40+
if iteration == 2 {
41+
debug!("Big file detected, the hash will take a while");
4742
}
43+
44+
length = {
45+
match reader.fill_buf(){
46+
Ok(buffer) =>{
47+
hasher.update(buffer);
48+
buffer.len()
49+
},
50+
Err(e) => {
51+
debug!("Cannot read file. Checksum set to 'UNKNOWN', error: {}", e);
52+
0
53+
}
54+
}
55+
};
56+
reader.consume(length);
57+
data_read += length / (1024 * 1024);
58+
iteration += 1;
4859
};
49-
reader.consume(length);
50-
data_read += length / (1024 * 1024);
51-
iteration += 1;
52-
};
53-
if data_read > read_limit {
60+
encode(hasher.finalize())
61+
}
62+
},
63+
Err(e) => {
64+
debug!("Cannot open file to get checksum, error: {:?}", e);
65+
String::from("UNKNOWN")
66+
}
67+
}
68+
}else{
69+
debug!("Cannot produce checksum of a removed file or directory.");
70+
String::from("UNKNOWN")
71+
}
72+
}
73+
74+
// ----------------------------------------------------------------------------
75+
76+
pub fn get_checksumv2<T: Digest>(filename: String, read_limit: usize, mut hasher: T) -> String {
77+
//let mut hasher = Sha3_512::new();
78+
let mut length = 1;
79+
let mut iteration = 0;
80+
let mut data_read = 0;
81+
let limit: usize = read_limit * 1024 * 1024;
82+
83+
if Path::new(&filename).is_file() {
84+
debug!("Getting hash of file: {}", filename);
85+
match File::open(filename.clone()){
86+
Ok(file) => {
87+
let size: usize = file.metadata().unwrap().len() as usize;
88+
let mut reader = BufReader::with_capacity(READ_CAPACITY, file);
89+
90+
91+
if size > limit {
5492
info!("File '{}' checksum skipped. File size is above limit.", filename);
5593
String::from("UNKNOWN")
5694
}else{
95+
while length > 0 && data_read <= limit {
96+
if iteration == 2 {
97+
debug!("Big file detected, the hash will take a while");
98+
}
99+
100+
length = {
101+
match reader.fill_buf(){
102+
Ok(buffer) =>{
103+
hasher.update(buffer);
104+
buffer.len()
105+
},
106+
Err(e) => {
107+
debug!("Cannot read file. Checksum set to 'UNKNOWN', error: {}", e);
108+
0
109+
}
110+
}
111+
};
112+
reader.consume(length);
113+
data_read += length;
114+
iteration += 1;
115+
};
57116
encode(hasher.finalize())
58117
}
59118
},

‎src/init.rs‎

Lines changed: 40 additions & 34 deletions
Original file line numberDiff line numberDiff line change
@@ -3,41 +3,47 @@
33
use crate::ruleset::Ruleset;
44
use crate::appconfig::*;
55
use crate::utils;
6+
use crate::db;
67

78

89
pub fn init() -> (AppConfig, Ruleset) {
9-
use std::path::Path;
10-
use simplelog::WriteLogger;
11-
use simplelog::Config;
12-
use std::fs;
13-
14-
println!("[INFO] Achiefs File Integrity Monitoring software starting!");
15-
println!("[INFO] Reading config...");
16-
let cfg = AppConfig::new(utils::get_os(), None);
17-
18-
// Create folders to store logs based on config.yml
19-
fs::create_dir_all(
20-
Path::new( &cfg.clone().log_file
21-
).parent().unwrap().to_str().unwrap()
22-
).unwrap();
23-
24-
// Create logger output to write generated logs.
25-
WriteLogger::init(
26-
cfg.clone().get_level_filter(),
27-
Config::default(),
28-
fs::OpenOptions::new()
29-
.create(true)
30-
.append(true)
31-
.open(cfg.clone().log_file)
32-
.expect("Unable to open log file")
33-
).unwrap();
34-
35-
println!("[INFO] Configuration successfully read, forwarding output to log file.");
36-
println!("[INFO] Log file: '{}'", cfg.clone().log_file);
37-
println!("[INFO] Log level: '{}'", cfg.clone().log_level);
38-
39-
let ruleset = Ruleset::new(utils::get_os(), None);
40-
41-
log_panics::init();
42-
(cfg, ruleset)
10+
use std::path::Path;
11+
use simplelog::WriteLogger;
12+
use simplelog::Config;
13+
use std::fs;
14+
15+
println!("[INFO] Achiefs File Integrity Monitoring software starting!");
16+
println!("[INFO] Reading config...");
17+
let cfg = AppConfig::new(utils::get_os(), None);
18+
19+
// Create folders to store logs based on config.yml
20+
fs::create_dir_all(
21+
Path::new( &cfg.clone().log_file
22+
).parent().unwrap().to_str().unwrap()
23+
).unwrap();
24+
25+
// Create logger output to write generated logs.
26+
WriteLogger::init(
27+
cfg.clone().get_level_filter(),
28+
Config::default(),
29+
fs::OpenOptions::new()
30+
.create(true)
31+
.append(true)
32+
.open(cfg.clone().log_file)
33+
.expect("Unable to open log file")
34+
).unwrap();
35+
36+
println!("[INFO] Configuration successfully read, forwarding output to log file.");
37+
println!("[INFO] Log file: '{}'", cfg.clone().log_file);
38+
println!("[INFO] Log level: '{}'", cfg.clone().log_level);
39+
40+
let ruleset = Ruleset::new(utils::get_os(), None);
41+
42+
let db = db::DB::new();
43+
db.create_table();
44+
println!("[INFO] Database created.");
45+
46+
println!("[INFO] Any error from this point will be logged in the log file.");
47+
log_panics::init();
48+
(cfg, ruleset)
4349
}

‎src/main.rs‎

Lines changed: 1 addition & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,6 @@ use std::sync::mpsc;
77
use std::thread;
88
use log::{error, info};
99
use crate::init::init;
10-
use crate::db::DBFile;
1110

1211
// Utils functions
1312
mod utils;
@@ -38,6 +37,7 @@ mod multiwatcher;
3837
mod rotator;
3938
mod init;
4039
mod db;
40+
mod scanner;
4141

4242
// ----------------------------------------------------------------------------
4343

@@ -77,17 +77,6 @@ async fn main() -> windows_service::Result<()> {
7777
Ok(_v) => info!("FIM rotator thread started."),
7878
Err(e) => error!("Could not start FIM rotator thread, error: {}", e)
7979
};
80-
let db = db::DB::new();
81-
db.create_table();
82-
db.insert_file(DBFile {
83-
id: 0, // Not used for insert auto-increment
84-
timestamp: String::from("Test"),
85-
hash: String::from("Test"),
86-
path: String::from("/test3"),
87-
size: 0
88-
});
89-
db.get_file(String::from("/test3"));
90-
db.print();
9180
monitor::monitor(tx, rx, cfg, ruleset).await;
9281
Ok(())
9382
},

‎src/monitor.rs‎

Lines changed: 10 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -9,13 +9,12 @@ use std::sync::mpsc;
99
use log::{info, error, debug, warn};
1010
// To manage paths
1111
use std::path::Path;
12-
// To manage date and time
13-
use std::time::{SystemTime, UNIX_EPOCH};
1412
use time::OffsetDateTime;
1513
// To use intersperse()
1614
use itertools::Itertools;
1715
// Event handling
1816
use notify::event::{EventKind, AccessKind};
17+
use sha3::{Digest, Sha3_512};
1918

2019

2120
// Utils functions
@@ -36,6 +35,7 @@ use crate::logreader;
3635
// integrations checker
3736
use crate::launcher;
3837
use crate::multiwatcher::MultiWatcher;
38+
use crate::scanner;
3939

4040
// ----------------------------------------------------------------------------
4141

@@ -130,7 +130,12 @@ pub async fn monitor(
130130
}
131131

132132
match watcher.watch(Path::new(path), RecursiveMode::Recursive) {
133-
Ok(_d) => debug!("Monitoring '{}' path.", path),
133+
Ok(_d) => {
134+
debug!("Monitoring '{}' path.", path);
135+
debug!("Starting file scan to create hash database.");
136+
scanner::scan_path(cfg.clone(), String::from(path));
137+
debug!("Path '{}' scanned all files are hashed in DB.", path);
138+
},
134139
Err(e) => warn!("Could not monitor given path '{}', description: {}", path, e)
135140
};
136141
}
@@ -206,7 +211,7 @@ pub async fn monitor(
206211

207212
let current_date = OffsetDateTime::now_utc();
208213
let index_name = format!("fim-{}-{}-{}", current_date.year(), current_date.month() as u8, current_date.day() );
209-
let current_timestamp = format!("{:?}", SystemTime::now().duration_since(UNIX_EPOCH).expect("Time went backwards").as_millis());
214+
let current_timestamp = utils::get_current_time_millis();
210215
let kind: notify::EventKind = event.kind;
211216
let path = event.paths[0].clone();
212217

@@ -284,7 +289,7 @@ pub async fn monitor(
284289
labels,
285290
operation: event::get_operation(kind),
286291
detailed_operation: event::get_detailed_operation(kind),
287-
checksum: hash::get_checksum( String::from(path.to_str().unwrap()), cfg.clone().events_max_file_checksum ),
292+
checksum: hash::get_checksum( String::from(path.to_str().unwrap()), cfg.clone().events_max_file_checksum, Sha3_512::new()),
288293
fpid: utils::get_pid(),
289294
system: cfg.clone().system
290295
};

0 commit comments

Comments
 (0)