Fix for zmap#913 and added IPv4 scan coverage integration test and python wrapper with --fast-dryrun (zmap#916)

* enable monitor thread and don't print packets on dryrun, for testing

* missed an int32

* Revert "enable monitor thread and don't print packets on dryrun, for testing"

This reverts commit c8bb512.

* Revert "Revert "enable monitor thread and don't print packets on dryrun, for testing""

This reverts commit 2976166.

* use monitor thread with dryrun but not receive thread

* print out packets to std out and also print status updates

* add fast dryrun option

* added fast-dryrun for increased performance, printing only dst IP and dst Port

* added a potentially working test, though it is FAR too slow to be usable for full IPv4 scans

* faster test, 2.5 Mpps

* unit tested verify fn

* working @ 2.56 Mpps

* speed up writing fast-dryrun

* better error handling

* eek'd out 25% more perf.

* added details on reproducing issue

* remove debugging logs and clean up

* off-by one error, candidate should be inclusive

* added a few more tests, added Dockerfile, and gh actions yml

* update manpages

* correct desc. of max_candidate

* cleanup

* added more integration tests for small subnets

* added test comments and cleaned up

* cleanup and make it more clear that we have a max_target and max_ip

* set top level dir correctly

* add dependency for bitarray

* handled starting monitor/recv threads in different CLI args combos

Author: phillip-stephens

.github/workflows/pytest.Dockerfile

@@ -2,7 +2,7 @@ FROM ubuntu:latest
 
 RUN apt-get update
 RUN apt-get install -y build-essential cmake libgmp3-dev gengetopt libpcap-dev flex \
-    byacc libjson-c-dev pkg-config libunistring-dev libjudy-dev cmake  make python3 python3-pytest python3-timeout-decorator curl
+    byacc libjson-c-dev pkg-config libunistring-dev libjudy-dev cmake  make python3 python3-pytest python3-timeout-decorator python3-bitarray curl
 RUN apt-get clean
 
 

.github/workflows/scan_coverage.yml

@@ -0,0 +1,32 @@
+name: Daily ZMap Coverage Test
+# This workflow is triggered  daily against main
+# It tests that in --fast-dryrun mode that ZMap scans all expected IPs+ports and doesn't scan targets multiple times
+# This test takes a while to run
+
+on:
+  # Allow manual triggering via the GitHub Actions UI
+  workflow_dispatch:
+
+  # Schedule the workflow to run once per day
+  schedule:
+    - cron: "0 0 * * *" # Adjust the time as needed (this example runs at midnight UTC)
+
+
+env:
+  ENABLE_DEVELOPMENT: ON
+  ENABLE_LOG_TRACE: ON
+  WITH_AES_HW: ON
+
+jobs:
+  example_job:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v3
+
+      - name: Run Scan Coverage Tests
+        run: |
+          docker build -t coverage-container -f .github/workflows/scan_coverage_pytest.Dockerfile .
+          docker run coverage-container
+          

.github/workflows/scan_coverage_pytest.Dockerfile

@@ -0,0 +1,20 @@
+FROM arm64v8/ubuntu:latest
+
+RUN apt-get update
+RUN apt-get install -y build-essential cmake libgmp3-dev gengetopt libpcap-dev flex byacc libjson-c-dev pkg-config \
+    libunistring-dev libjudy-dev cmake  make python3 python3-pytest python3-timeout-decorator python3-bitarray curl
+RUN apt-get clean
+
+
+
+WORKDIR /zmap
+COPY . .
+RUN rm -f CMakeCache.txt # if building locally, this file can be present and cause build failure
+RUN cmake -DENABLE_DEVELOPMENT=$ENABLE_DEVELOPMENT -DENABLE_LOG_TRACE=$ENABLE_LOG_TRACE -DWITH_AES_HW=$WITH_AES_HW . \
+    && make -j4
+
+WORKDIR /zmap/test/integration-tests
+# need to get the gateway mac
+RUN curl zmap.io \
+    # launch the test
+    && python3 test_full_ipv4_multi_port_scan.py

src/cyclic.c

@@ -58,7 +58,7 @@ static cyclic_group_t groups[] = {
     {// 2^8 + 1
      .prime = 257,
      .known_primroot = 3,
-     .prime_factors = {2},
+     .prime_factors = {2}, // prime factors of prime - 1
      .num_prime_factors = 1},
     {// 2^16 + 1
      .prime = 65537,

src/iterator.c

@@ -83,11 +83,17 @@ iterator_t *iterator_init(uint8_t num_threads, uint16_t shard,
 	iterator_t *it = xmalloc(sizeof(struct iterator));
 	const cyclic_group_t *group = get_group(group_min_size);
 	if (num_addrs > (1LL << 32)) {
-		zsend.max_index = 0xFFFFFFFF;
+		zsend.max_ip_index = 0xFFFFFFFF;
 	} else {
-		zsend.max_index = (uint32_t)num_addrs;
+		zsend.max_ip_index = (uint32_t)num_addrs;
 	}
-	log_debug("iterator", "max index %u", zsend.max_index);
+	log_debug("iterator", "max ip index %ul", zsend.max_ip_index);
+	// The candidate is upper-bounded by the modulus of the group, which is the prime number chosen in cyclic.c.
+	// All primes are chosen to be greater than the number of allowed targets.
+	// We must re-roll if the candidate target is out of bounds of (2 ** 32) * (2 ** number of ports).
+	zsend.max_target_index = 1ULL << (32 + bits_for_port);
+	log_debug("iterator", "max target index %ull", zsend.max_target_index);
+
 	it->cycle = make_cycle(group, zconf.aes);
 	it->num_threads = num_threads;
 	it->curr_threads = num_threads;

src/monitor.c

@@ -152,11 +152,11 @@ double compute_remaining_time(double age, uint64_t packets_sent,
 			    (double)zrecv.filter_success / zconf.max_results;
 			remaining[3] = (1. - done) * (age / done);
 		}
-		if (zsend.max_index) {
+		if (zsend.max_ip_index) {
 			double done =
 			    (double)packets_sent /
-			    ((uint64_t)zsend.max_index * zconf.ports->port_count * zconf.packet_streams /
-			     zconf.total_shards);
+			    ((uint64_t)zsend.max_ip_index * zconf.ports->port_count * zconf.packet_streams /
+				 zconf.total_shards);
 			remaining[4] =
 			    (1. - done) * (age / done) + zconf.cooldown_secs;
 		}

src/probe_modules/module_tcp_synscan.c

@@ -137,6 +137,16 @@ void synscan_print_packet(FILE *fp, void *packet)
 	struct ether_header *ethh = (struct ether_header *)packet;
 	struct ip *iph = (struct ip *)&ethh[1];
 	struct tcphdr *tcph = (struct tcphdr *)&iph[1];
+	if (zconf.fast_dryrun) {
+		// We'll just print a binary representation of the dst IP and the dst Port to reduce data output/save time
+		struct in_addr *dest_IP = (struct in_addr *)&(iph->ip_dst);
+		// Writing binary IP addresses
+		const uint8_t IP_ADDR_LEN = 4;
+		const uint8_t TCP_PORT_LEN = 2;
+		fwrite(&(dest_IP->s_addr),  IP_ADDR_LEN,1, fp);  // Write destination IP (binary)
+		fwrite(&(tcph->th_dport),  TCP_PORT_LEN,1, fp);  // Write destination port (binary)
+		return;
+	}
 	fprintf(fp,
 		"tcp { source: %u | dest: %u | seq: %u | checksum: %#04X }\n",
 		ntohs(tcph->th_sport), ntohs(tcph->th_dport),

src/send.c

@@ -276,7 +276,11 @@ int send_run(sock_t st, shard_t *s)
 				   20;
 			last_time = steady_now();
 			assert(interval > 0);
-			assert(delay > 0);
+			if (delay == 0) {
+				// at extremely high bandwidths, the delay could be set to zero.
+				// this breaks the multiplier logic below, so we'll hard-set it to 1 in this case.
+				delay = 1;
+			}
 		}
 	}
 	int attempts = zconf.retries + 1;
@@ -413,10 +417,17 @@ int send_run(sock_t st, shard_t *s)
 			batch->packets[batch->len].len = (uint32_t)length;
 
 			if (zconf.dryrun) {
-				lock_file(stdout);
-				zconf.probe_module->print_packet(stdout,
-								 batch->packets[batch->len].buf);
-				unlock_file(stdout);
+				batch->len++;
+				if (batch->len == batch->capacity) {
+					lock_file(stdout);
+					for (int i = 0; i < batch->len; i++) {
+						zconf.probe_module->print_packet(stdout,
+														 batch->packets[i].buf);
+					}
+					unlock_file(stdout);
+					// reset batch length for next batch
+					batch->len = 0;
+				}
 			} else {
 				batch->len++;
 				if (batch->len == batch->capacity) {
@@ -465,6 +476,15 @@ int send_run(sock_t st, shard_t *s)
 cleanup:
 	if (!zconf.dryrun && send_batch(st, batch, attempts) < 0) {
 		log_error("send_batch cleanup", "could not send remaining batch packets: %s", strerror(errno));
+	} else if (zconf.dryrun) {
+		lock_file(stdout);
+		for (int i = 0; i < batch->len; i++) {
+			zconf.probe_module->print_packet(stdout,
+											 batch->packets[i].buf);
+		}
+		unlock_file(stdout);
+		// reset batch length for next batch
+		batch->len = 0;
 	}
 	free_packet_batch(batch);
 	s->cb(s->thread_id, s->arg);

src/shard.c

@@ -32,7 +32,7 @@ static void shard_roll_to_valid(shard_t *s)
 {
 	uint64_t current_ip_index = (s->current - 1) >> s->bits_for_port;
 	uint16_t candidate_port = extract_port(s->current - 1, s->bits_for_port);
-	if (current_ip_index < zsend.max_index && candidate_port < zconf.ports->port_count) {
+	if (current_ip_index < zsend.max_ip_index && candidate_port < zconf.ports->port_count) {
 		return;
 	}
 	shard_get_next_target(s);
@@ -105,7 +105,6 @@ void shard_init(shard_t *shard, uint16_t shard_idx, uint16_t num_shards,
 	shard->params.last = (uint64_t)mpz_get_ui(stop_m);
 	shard->params.factor = cycle->generator;
 	shard->params.modulus = cycle->group->prime;
-	//
 	shard->bits_for_port = bits_for_port;
 
 	// Set the shard at the beginning.
@@ -159,7 +158,7 @@ static inline uint64_t shard_get_next_elem(shard_t *shard)
 {
 	shard->current *= shard->params.factor;
 	shard->current %= shard->params.modulus;
-	return (uint64_t)shard->current;
+	return shard->current;
 }
 
 target_t shard_get_next_target(shard_t *shard)
@@ -176,11 +175,17 @@ target_t shard_get_next_target(shard_t *shard)
 			return (target_t){
 			    .ip = 0, .port = 0, .status = ZMAP_SHARD_DONE};
 		}
+		if (candidate >= zsend.max_target_index) {
+			// If the candidate is out of bounds, re-roll. This will happen since we choose primes/moduli that are
+			// larger than the number of allowed targets. The IP is bounded below by checking against zsend.max_ip_index.
+			continue;
+		}
+		// Good candidate, proceed with it.
 		uint32_t candidate_ip =
 		    extract_ip(candidate - 1, shard->bits_for_port);
 		uint16_t candidate_port =
 		    extract_port(candidate - 1, shard->bits_for_port);
-		if (candidate_ip < zsend.max_index &&
+		if (candidate_ip < zsend.max_ip_index &&
 		    candidate_port < zconf.ports->port_count) {
 			shard->iterations++;
 			return (target_t){

src/state.c

@@ -24,6 +24,7 @@ struct state_conf zconf = {
     .dedup_method = 0,
     .dedup_window_size = 0,
     .dryrun = 0,
+    .fast_dryrun = 0,
     .hw_mac = {0},
     .hw_mac_set = 0,
     .gw_ip = 0,