added some wiki processing changes

Author: thejourneyler

Data-Source/Data-Wiki/ful/AA/wiki_00

@@ -79,24 +79,28 @@ pushd $DIR_WIKI_DATA
 find * -maxdepth 0 -type d  \( ! -iname ".*" \) >> "$HOME_FOLDER"/$WIKI_LANGUAGES
 popd
 
+# For each item/line in Wiki_languages find out if the line already exists in $LANGUAGE_LIST_FILE and if not append it.
+
 for i in $(cat $WIKI_LANGUAGES);do
-	grep -Fxq "$i" $LANGUAGE_LIST_FILE || echo $i >> $LANGUAGE_LIST_FILE
-	grep -Fxq "$i" $CORPORA_LANGUAGES || echo $i >> $CORPORA_LANGUAGES
+	grep -Fxq "$i" $LANGUAGE_LIST_FILE || echo "$i" >> $LANGUAGE_LIST_FILE #There is a bug here and I can not seem to pass data into this$LANGUAGE_LIST_FILE.
+	grep -Fxq "$i" $CORPORA_LANGUAGES || echo "$i" >> $CORPORA_LANGUAGES
 done
 
 
 # Set the Variables.
+
+#turn the list into a long list (array) with out new lines instead of a tall list
 WIKI_LANGUAGESString=$(cat $WIKI_LANGUAGES | tr "\n" " ")
-WIKI_LANGUAGES_ARRAY=($WIKI_LANGUAGESString) #There is a bug here (or at least a bad programming practice). The file veriable has one name and the same name is used later for a different meaning. Fixed on 15 July 2015 by adding "_ARRAY at the end of the variable name".
+WIKI_LANGUAGES_ARRAY=($WIKI_LANGUAGESString)
 
 # This section needs to be modified and allow the arangement of info
 # to be corpus by type: Wikpedia/James or Language Navajo/ibgo
 
-echo "INFO: It looks like we were able to extract ${#WIKI_LANGUAGES_ARRAY[@]} Wikipedia based corpora."
+echo "INFO: It looks like we were able to extract ${#WIKI_LANGUAGES_ARRAY[@]} Wikipedia based corpora." #There is a bug here and I can\'t seem to find out why the data is not being passed correctly. The same thing is happening in James. 
 echo "      Including the following languages: ${WIKI_LANGUAGES_ARRAY[*]}"
 echo
 
-exit;0
+
 
 # Take the languages from Wikipedia and append them to the master language list; making sure not to add duplicates
 
@@ -108,8 +112,8 @@ exit;0
 #   Some_third_array_count=$((${#Some_array[*]} + ${#Some_other_array[*]}))
 
 
-csvfix unique Wikipedia_Languages.txt $LANGUAGE_LIST_FILE | csvfix write_dsv -s ' ' -o $LANGUAGE_LIST_FILE
-
+csvfix unique /Temp-Files/Languages-Used/Wikipedia_Languages.txt $LANGUAGE_LIST_FILE | csvfix write_dsv -s ' ' -o $LANGUAGE_LIST_FILE
+exit;0
 
 # Set the Variables.
 LANGUAGE_IDString=$(cat $LANGUAGE_LIST_FILE | tr "\n" " ")

Data-Source/Data-Wiki/hau/AA/wiki_00

@@ -170,6 +170,7 @@ for i in $(cat $JAMES_LIST_FILE); do
 done
 
 # Take the languages from James and add them to the master language list.
+echo $(cat $JAMES_LANGUAGES)
 for i in $(cat $JAMES_LANGUAGES);do
 	grep -Fxq "$i" $LANGUAGE_LIST_FILE || echo $i >> $LANGUAGE_LIST_FILE
 	grep -Fxq "$i" $CORPORA_LANGUAGES || echo $i >> $CORPORA_LANGUAGES
@@ -181,9 +182,8 @@ done
 JAMES_LANGUAGESString=$(cat $JAMES_LANGUAGES | tr "\n" " ")
 JAMES_LANGUAGES_ARRAY=($JAMES_LANGUAGESString) #There is a bug here (or at least a bad programming practice). The file veriable has one name and the same name is used later for a different meaning. Fixed on 15 July 2015 by adding "_ARRAY at the end of the variable name".
 
-
 ## These are not reading as arrays. Rather they are reading as literals or paths OR they are reading as just the first line of the file.
-echo "INFO: It looks like altogether we found: ${#JAMES_LANGUAGES_ARRAY[@]} James based corpora."
+echo "INFO: It looks like altogether we found: ${#JAMES_LANGUAGESString[@]} James based corpora."
 echo "      Including the following languages: ${JAMES_LANGUAGES_ARRAY[*]}"
 echo