Refactored DHS data import.

Removed data from surveys and imported subnational data in data collection: this way we can compute the data we are interested in and set it into the surveys.

Author: milko

.idea/workspace.xml

@@ -1084,9 +1084,9 @@ public function InitSurveys()
 				//
 				// Load survey data.
 				//
-				$data_points = $this->loadSurveyData( $line[ 'SurveyId' ] );
-				if( count( $data_points ) )
-					$document[ kTAG_DATA ] = $data_points;
+//				$data_points = $this->loadSurveyData( $line[ 'SurveyId' ] );
+//				if( count( $data_points ) )
+//					$document[ kTAG_DATA ] = $data_points;
 
 				//
 				// Save document.
@@ -1135,9 +1135,9 @@ public function InitSurveys()
 	 *==================================================================================*/
 
 	/**
-	 * <h4>Initialise surveys.</h4>
+	 * <h4>Initialise survey data.</h4>
 	 *
-	 * This method will load the surveys.
+	 * This method will load the surveys data at sub-national level.
 	 */
 	public function InitData()
 	{
@@ -1146,8 +1146,8 @@ public function InitData()
 		//
 		$collection = $this->mDatabase->NewDataCollection();
 		$page = 1;
-		$lines = 100;
-		$url = self::kDHS_URL_DATA . "&page=$page&perpage=$lines";
+		$lines = 600;
+		$url = self::kDHS_URL_DATA . "subnational?f=json&page=$page&perpage=$lines";
 
 		//
 		// Read data.
@@ -1185,8 +1185,11 @@ public function InitData()
 				//
 				// Set identifiers.
 				//
-				$document[ $collection->KeyOffset() ]
-					= $line[ 'SurveyId' ] . ':' . $line[ 'DataId' ];
+				$document[ $collection->KeyOffset() ] = (int)$line[ 'DataId' ];
+//				$document[ $collection->KeyOffset() ]
+//					= $line[ 'SurveyId' ] . ':'
+//					. $line[ 'RegionId' ] . ':'
+//					. $line[ 'DataId' ];
 
 				//
 				// Set other data.
@@ -1252,7 +1255,7 @@ public function InitData()
 			// Get next.
 			//
 			$page++;
-			$url = self::kDHS_URL_DATA . "&page=$page&perpage=$lines";
+			$url = self::kDHS_URL_DATA . "subnational?f=json&page=$page&perpage=$lines";
 			$retries = self::kRETRIES;
 			while( $retries-- )
 			{

batch/DHS.php

@@ -18,7 +18,7 @@
  * Initialisation definition.
  */
 define( 'doINIT', TRUE );
-define( 'kENGINE', 'ARANGO' );
+define( 'kENGINE', 'MONGO' );
 
 /*
  * Global includes.
@@ -143,9 +143,9 @@
 //
 // Initialise DHS data.
 //
-//echo( "- Initialising DHS data: " );
-//$dhs->InitData();
-//echo( " Done.\n" );
+echo( "- Initialising DHS data: " );
+$dhs->InitData();
+echo( " Done.\n" );
 
 
 ?>