Added aggregation pipelines and added "regional" breajdown.

Set the "regional" breakdown category to all "background" breakdowns that had "region" as their category.
Added a set of aggregation pipelines to check for duplicates and other stuff.
Need to study the actual surveys, since this data is mostly summary.
Will launch ArangoDB import.

Author: milko

.idea/workspace.xml

@@ -1262,7 +1262,11 @@ public function loadSurveyData( $theSurvey )
 					//
 					// Set descriptor breakdown.
 					//
-					$document[ kTAG_BREAKDOWN ] = $domain;
+					$document[ kTAG_BREAKDOWN ]
+						= ( strlen( trim( $line[ 'RegionId' ] ) )
+						 && (trim( $line[ 'CharacteristicCategory' ] ) == 'Region') )
+						? 'regional'
+						: $domain;
 
 					//
 					// Set other data.

batch/DHS.php

@@ -0,0 +1,37 @@
+db._data.aggregate(
+
+  // Pipeline
+  [
+    // Stage 1
+    {
+      $group: {
+      	_id: "$@b",
+      	"count": { $sum: 1 }
+      }
+    },
+
+    // Stage 2
+    {
+      $match: {
+      	"count": { $gt: 1 }
+      }
+    },
+
+    // Stage 3
+    {
+      $out: "checks_DuplicateDataIdentifiers"
+    }
+  ],
+
+  // Options
+  {
+    cursor: {
+      batchSize: 50
+    },
+
+    allowDiskUse: true
+  }
+
+  // Created with 3T MongoChef, the GUI for MongoDB - http://3t.io/mongochef
+
+);