Fix index tuning

andrewvc · May 26, 2015 · 49ecfe6 · 49ecfe6
1 parent 5f193bb
commit 49ecfe6
Show file tree

Hide file tree

Showing 2 changed files with 17 additions and 16 deletions.
diff --git a/project.clj b/project.clj
@@ -1,4 +1,4 @@
-(defproject wikiparse "1.2.4"
+(defproject wikiparse "1.2.5"
   :description "Import Wikipedia data into elasticsearch"
   :url "http://example.com/FIXME"
   :aot [wikiparse.core]

diff --git a/src/wikiparse/core.clj b/src/wikiparse/core.clj
@@ -14,19 +14,6 @@
 
 (def connection (atom nil))
 
-(defn tune-performance
-  [conn name]
-  (println "Tuning index performance")
-  (es-index/update-settings conn name {:index
-                                       {
-                                        :refresh_interval 60
-                                        :gateway {:local {:sync "60s"}}
-                                        :translog {
-                                                   :interval "60s"
-                                                   :flush_threshold_size "756mb"
-                                                   }
-                                        } } ))
-
 (defn connect!
   "Connect once to the ES cluster"
   [es]
@@ -200,7 +187,22 @@
     (println (format "Deleting index %s" name))
     (es-index/delete conn name)
     (println (format "Creating index %s" name))
-    (es-index/create conn name :settings {:index {:number_of_shards 1, :number_of_replicas 0}} :mappings {:page page-mapping})))
+    (es-index/create conn name
+                     :settings {
+                                :index {
+                                        :number_of_shards 1,
+                                        :number_of_replicas 0
+                                        :refresh_interval 60
+                                        :gateway {:local {:sync "60s"}}
+                                        :translog {
+                                                   :interval "60s"
+                                                   :flush_threshold_size "756mb"
+                                                   }
+                                        }
+                                }
+                     :mappings {
+                                :page page-mapping
+                                })))
 
 (defn index-dump
   [rdr conn callback phase index-name batch-size]
@@ -264,7 +266,6 @@
   (let [[opts path] (parse-cmdline args)]
     (with-connection (:es opts) conn
                      (ensure-index conn (:index opts))
-                     (tune-performance conn (:index opts))
                      (doseq [phase (map keyword (string/split (:phases opts) #","))]
                        (let [stats (swap! phase-stats (fn [_] (new-phase-stats phase)))
                              batch-size (Integer/parseInt (:batch opts))