Add parameter search code and features (landmarks)

rpi-trust · Apr 6, 2020 · daad6b3 · daad6b3
1 parent b5e901f
commit daad6b3
Show file tree

Hide file tree

Showing 160 changed files with 5,480,704 additions and 23,204 deletions.
diff --git a/features/english/cosine.csv b/features/english/cosine.csv
diff --git a/features/english/map.csv b/features/english/map.csv
diff --git a/features/landmarks/english/100/cosine.csv b/features/landmarks/english/100/cosine.csv
diff --git a/features/landmarks/english/100/freq_diff.csv b/features/landmarks/english/100/freq_diff.csv
diff --git a/features/landmarks/english/100/map.csv b/features/landmarks/english/100/map.csv
diff --git a/features/landmarks/english/10000/cosine.csv b/features/landmarks/english/10000/cosine.csv
diff --git a/features/landmarks/english/10000/freq_diff.csv b/features/landmarks/english/10000/freq_diff.csv
diff --git a/features/landmarks/english/10000/map.csv b/features/landmarks/english/10000/map.csv
diff --git a/features/landmarks/english/1500/cosine.csv b/features/landmarks/english/1500/cosine.csv
diff --git a/features/landmarks/english/1500/freq_diff.csv b/features/landmarks/english/1500/freq_diff.csv
diff --git a/features/landmarks/english/1500/map.csv b/features/landmarks/english/1500/map.csv
diff --git a/features/landmarks/english/200/cosine.csv b/features/landmarks/english/200/cosine.csv
diff --git a/features/landmarks/english/200/freq_diff.csv b/features/landmarks/english/200/freq_diff.csv
diff --git a/features/landmarks/english/200/map.csv b/features/landmarks/english/200/map.csv
diff --git a/features/landmarks/english/2100/cosine.csv b/features/landmarks/english/2100/cosine.csv
diff --git a/features/landmarks/english/2100/freq_diff.csv b/features/landmarks/english/2100/freq_diff.csv
diff --git a/features/landmarks/english/2100/map.csv b/features/landmarks/english/2100/map.csv
diff --git a/features/landmarks/english/2700/cosine.csv b/features/landmarks/english/2700/cosine.csv
diff --git a/features/landmarks/english/2700/freq_diff.csv b/features/landmarks/english/2700/freq_diff.csv
diff --git a/features/landmarks/english/2700/map.csv b/features/landmarks/english/2700/map.csv
diff --git a/features/landmarks/english/300/cosine.csv b/features/landmarks/english/300/cosine.csv
diff --git a/features/landmarks/english/300/freq_diff.csv b/features/landmarks/english/300/freq_diff.csv
diff --git a/features/landmarks/english/300/map.csv b/features/landmarks/english/300/map.csv
diff --git a/features/landmarks/english/3000/cosine.csv b/features/landmarks/english/3000/cosine.csv
diff --git a/features/landmarks/english/3000/freq_diff.csv b/features/landmarks/english/3000/freq_diff.csv
diff --git a/features/landmarks/english/3000/map.csv b/features/landmarks/english/3000/map.csv
diff --git a/features/landmarks/english/4000/cosine.csv b/features/landmarks/english/4000/cosine.csv
diff --git a/features/landmarks/english/4000/freq_diff.csv b/features/landmarks/english/4000/freq_diff.csv
diff --git a/features/landmarks/english/4000/map.csv b/features/landmarks/english/4000/map.csv
diff --git a/features/landmarks/english/5000/cosine.csv b/features/landmarks/english/5000/cosine.csv
diff --git a/features/landmarks/english/5000/freq_diff.csv b/features/landmarks/english/5000/freq_diff.csv
diff --git a/features/landmarks/english/5000/map.csv b/features/landmarks/english/5000/map.csv
diff --git a/features/landmarks/english/900/cosine.csv b/features/landmarks/english/900/cosine.csv
diff --git a/features/landmarks/english/900/freq_diff.csv b/features/landmarks/english/900/freq_diff.csv
diff --git a/features/landmarks/english/900/map.csv b/features/landmarks/english/900/map.csv
diff --git a/features/landmarks/fix_filenames.sh b/features/landmarks/fix_filenames.sh
@@ -0,0 +1,25 @@
+
+#!bin/bash
+declare -A news_groups
+news_groups=( ["fasttext_300_Infowars-The Gateway Pundit-Veterans Today-Newswars-Prison Planet"]="news_group1"
+        ["fasttext_300_National Review-Fox News-Real Clear Politics-Washington Examiner-Investors Business Daily"]="news_group4"
+        ["fasttext_300_Reuters-CNN-CBS News-ABC News-The Hill"]="news_group3"
+        ["fasttext_300_RT-Sputnik-Tass-Russia-Insider-Fort Russ"]="news_group6"
+        ["fasttext_300_Talking Points Memo-Raw Story-Daily Kos-Crooks and Liars-Media Matters for America"]="news_group2"
+        ["fasttext_300_The Daily Caller-Breitbart-RedState-Hot Air-TheBlaze"]="news_group5"
+        ["fasttext_300_The Guardian UK-The Independent-Evening Standard-BBC UK-The Daily Mirror-The Sun-Birmingham Mail"]="news_group7")
+
+parent_dir="raw"
+for i in $parent_dir/*; do
+  for j in "$i"/*.txt ;
+    do mv -- "$j" "${j%.txt}.vec";
+  done  ;
+done
+
+
+for i in $parent_dir/*; do
+  for j in "$i"/*.vec ; do
+    echo $(basename "$j") "${news_groups[$(basename "${j%.vec}")]}"
+    mv -- "$j" $(dirname "$j")/"${news_groups["$(basename "${j%.vec}")"]}.vec"
+  done
+done