add --show-real-ips logic and convert to async mode

src/consts.nim

@@ -0,0 +1,20 @@
+const
+  CF_IP_API: string = "https://api.cloudflare.com/client/v4/ips"
+  CF_REAL_IP_HEADER: string = "CF-Connecting-IP;"
+
+  FASTLY_IP_API:string = "https://api.fastly.com/public-ip-list"
+
+  DEFAULT_OUTPUT_PATH: string = "/etc/nginx/reverse_proxies"
+
+  CFG_SET_REAL_IP_FROM: string = "set_real_ip_from"
+  CFG_REAL_IP_HEADER: string = "real_ip_header"
+
+  ONE_MINUTE: int = 60_000
+  THREE_HOURS: int = 108_000_00
+  SIX_HOURS: int = 216_000_00
+  TWELVE_HOURS: int = 432_000_00
+  # TWELVE_HOURS: int = int(12.hours.milliseconds)
+
+  NGINX_PROCESS_NAME: string = "nginx"
+  NGINX_TEST_CMD: string = "nginx -t"
+  NGINX_RELOAD_CMD: string = "nginx -s reload"

src/nginwho.nim

@@ -1,13 +1,14 @@
 import std/[strutils, os, strformat, re]
 import db_connector/db_sqlite
 from parseopt import CmdLineKind, initOptParser, next
+include realip
 
 
 const
     NGINX_DEFAULT_LOG_PATH = "/var/log/nginx/access.log"
     DB_FILE = "/var/log/nginwho.db"
-    TEN_SECONDS = 10000
-    VERSION = "0.5.0"
+    TEN_SECONDS = 10_000
+    VERSION = "0.7.0"
 
 
 type Log = object
@@ -29,20 +30,22 @@ type
     logPath: string,
     dbPath: string,
     interval: int,
-    omitReferrer: string
+    omitReferrer: string,
+    showRealIPs: bool
   ]
   Logs = seq[Log]
 
 
 proc usage() =
   echo """
 
-  --help, -h         : show help
+  --help, -h         : Show help
   --version, -v      : Display version and quit
   --dbPath,          : Path to SQLite database to log reports (default: /var/log/nginwho.db)
   --logPath,         : Path to nginx access logs (default: /var/log/nginx/access.log)
   --interval         : Refresh interval in seconds (default: 10)
-  --omit-referrer    : omit a specific referrer from being logged
+  --omit-referrer    : Omit a specific referrer from being logged
+  --show-real-ips    : Show real IP of visitors by getting Cloudflare CIDRs to include in nginx config. Self-updates every six hours.
 
   """
   quit()
@@ -53,7 +56,8 @@ proc getArgs(): Flags =
       logPath: NGINX_DEFAULT_LOG_PATH,
       dbPath:DB_FILE,
       interval: TEN_SECONDS,
-      omitReferrer: ""
+      omitReferrer: "",
+      showRealIPs: false
     )
 
   var p = initOptParser()
@@ -70,14 +74,11 @@ proc getArgs(): Flags =
         quit(0)
       of "logPath": flags.logPath = p.val
       of "dbPath": flags.dbPath = p.val
-      of "interval": flags.interval = parseInt(p.val)
+      of "interval": flags.interval = parseInt(p.val) * 1000 # convert to seconds
       of "omit-referrer": flags.omitReferrer = p.val
+      of "show-real-ips": flags.showRealIPs = parseBool(p.val)
     of cmdArgument: discard
 
-  if flags.dbPath == "": flags.dbPath = DB_FILE
-  if flags.logPath == "": flags.logPath = NGINX_DEFAULT_LOG_PATH
-  if flags.interval == 0: flags.interval = TEN_SECONDS
-
   return flags
 
 
@@ -175,13 +176,7 @@ proc parseLogEntry(logLine: string, omit: string): Log =
   return log
 
 
-proc main() = 
-  let args: Flags = getArgs()
-
-  if not fileExists(args.logPath):
-    echo fmt"File not found: {args.logPath}"
-    quit(1)
-
+proc processLogs(args: Flags) {.async.} =
   let db: DbConn = open(args.dbPath, "", "", "")
   defer: db.close()
 
@@ -193,8 +188,24 @@ proc main() =
       logs.add(log)
 
     writeToDatabase(logs, db)
-    sleep args.interval
-
+
+    await sleepAsync args.interval
+
+
+proc main() = 
+  let args: Flags = getArgs()
+
+  if not fileExists(args.logPath):
+    echo fmt"File not found: {args.logPath}"
+    quit(1)
+
+  if args.showRealIPs:
+    echo "Scheduling CIDRs retrieval to display real visitors IPs"
+    asyncCheck fetchAndProcessIPCidrs()
+
+  asyncCheck processLogs(args)
+
+  runForever()
 
 when is_main_module:
   main()

src/nginx.nim

@@ -0,0 +1,27 @@
+import std/[os, osproc, strformat, times]
+
+
+proc ensureNginxExists() =
+  let now: string = getTime().format("yyyy-MM-dd HH:mm:ss")
+
+  let result: string = findExe(NGINX_PROCESS_NAME)
+  if result == "":
+    quit(fmt"{now} - nginx command not found", 1)
+
+
+proc testNginxConfig(): int =
+  let now: string = getTime().format("yyyy-MM-dd HH:mm:ss")
+  echo fmt"{now} - Testing nginx configuration"
+
+  return execCmd(command=NGINX_TEST_CMD)
+
+
+proc reloadNginx() =
+  let now: string = getTime().format("yyyy-MM-dd HH:mm:ss")
+
+  let result: int = execCmd(command=NGINX_RELOAD_CMD)
+  if result != 0:
+    echo fmt"{now} - nginx process reload failed"
+  else:
+    echo fmt"{now} - nginx process reloaded successfully"
+

src/realip.nim

@@ -0,0 +1,117 @@
+import std/[asyncdispatch, httpclient, json, strformat, options, strutils]
+include consts, nginx
+
+
+type
+  IPCidrs = object
+    ipv4: JsonNode
+    ipv6: JsonNode
+    etag: string
+    etagChanged: bool
+
+
+proc reloadNginxAt(hour: int = 3, minute: int = 0) {.async.} =
+  while true:
+    let now: DateTime = getTime().local()
+    if now.hour == hour and now.minute == minute:
+      let testResult = testNginxConfig()
+      if testResult != 0:
+        echo fmt"{now} - nginx configuration test failed"
+        continue
+
+      reloadNginx()
+
+    await sleepAsync(ONE_MINUTE)
+
+
+proc populateReverseProxyFile(filePath: string=DEFAULT_OUTPUT_PATH, ipCidr: IPCidrs): bool =
+  let now: string = getTime().format("yyyy-MM-dd HH:mm:ss")
+  echo fmt"{now} - Populating CIDRs file in {filePath}"
+
+  if ipCidr.etagChanged:
+    try:
+      let file: File = open(filePath, fmWrite)
+      defer: file.close()
+
+      file.write("# Cloudflare ranges\n")
+      file.write("# Last etag: ", ipCidr.etag, "\n\n")
+      file.write("# IPv4 CIDRS\n")
+
+      for cidr in ipCidr.ipv4:
+        file.write(CFG_SET_REAL_IP_FROM, " ", cidr.getStr(), ";", "\n")
+
+      file.write("\n# IPv6 CIDRS\n")
+
+      for cidr in ipCidr.ipv6:
+        file.write(CFG_SET_REAL_IP_FROM, " ", cidr.getStr(), ";", "\n")
+
+      file.write("\n\n", CFG_REAL_IP_HEADER, " ", CF_REAL_IP_HEADER, "\n")
+      return true
+    except:
+      echo fmt"{now} - Could not open {filePath}"
+      return false
+
+
+proc getCloudflareCIDRs(): Option[IPCidrs] =
+  let now: string = getTime().format("yyyy-MM-dd HH:mm:ss")
+  echo fmt"{now} - Getting Cloudflare CIDRs"
+
+  let client: HttpClient = newHttpClient()
+  let response: Response = client.get(CF_IP_API)
+
+  if response.code != Http200:
+    echo fmt"{now} - API call to {CF_IP_API} failed"
+    return none(IPCidrs)
+
+  let jsonResponse: JsonNode = parseJson(response.body)
+
+  let etag: string =  jsonResponse["result"]["etag"].getStr()
+
+  let apiSuccess: bool =  jsonResponse["success"].getBool()
+  if apiSuccess != true:
+    echo fmt"{now} - API `success` is not true: {apiSuccess}"
+    return none(IPCidrs)
+
+  let ipv4Cidrs: JsonNode =  jsonResponse["result"]["ipv4_cidrs"]
+  let ipv6Cidrs: JsonNode = jsonResponse["result"]["ipv6_cidrs"]
+
+  if ipv4Cidrs.isNil or ipv6Cidrs.isNil:
+    return none(IPCidrs)
+  else:
+    return some(IPCidrs(ipv4: ipv4Cidrs, ipv6: ipv6Cidrs, etag: etag, etagChanged: true))
+
+
+proc getCurrentEtag(configFile: string=DEFAULT_OUTPUT_PATH): string = 
+  let now: string = getTime().format("yyyy-MM-dd HH:mm:ss")
+
+  if not fileExists(configFile):
+    echo fmt"{now} - {configFile} does not exist"
+    return
+
+  for line in lines(configFile):
+    if line.startsWith("# Last etag:"):
+      let etagLine: seq[string] = line.split("# Last etag: ")
+      if len(etagLine) > 1:
+        return etagLine[1]
+
+
+proc fetchAndProcessIPCidrs() {.async.} =
+  let now: string = getTime().format("yyyy-MM-dd HH:mm:ss")
+
+  while true:
+    let etag: string = getCurrentEtag()
+    let cfCIDRs: Option[IPCidrs] = getCloudflareCIDRs()
+
+    case cfCIDRs.isSome:
+    of true:
+      let cidrs: IPCidrs = cfCIDRs.get()
+      if etag != cidrs.etag:
+        if populateReverseProxyFile(ipCidr=cidrs):
+          waitFor reloadNginxAt()
+      else:
+          echo fmt"{now} - etag has not changed {etag}"
+    of false:
+      echo fmt"{now} - Failed fetching CIDRs"
+
+    await sleepAsync(SIX_HOURS)
+