Add locales support

CHANGELOG.md

@@ -1,3 +1,6 @@
+## 6.0.4
+  - Add locales support for output fields, such as `country_name` and `city_name` [#160](https://github.com/logstash-plugins/logstash-filter-geoip/pull/160)
+
 ## 6.0.3
   - Fixed docs for missing region_code [#158](https://github.com/logstash-plugins/logstash-filter-geoip/pull/158)
 

docs/index.asciidoc

@@ -74,6 +74,7 @@ This plugin supports the following configuration options plus the <<plugins-{typ
 | <<plugins-{type}s-{plugin}-database>> |a valid filesystem path|No
 | <<plugins-{type}s-{plugin}-default_database_type>> |`City` or `ASN`|No
 | <<plugins-{type}s-{plugin}-fields>> |<<array,array>>|No
+| <<plugins-{type}s-{plugin}-locales>> |<<array,array>>|No
 | <<plugins-{type}s-{plugin}-source>> |<<string,string>>|Yes
 | <<plugins-{type}s-{plugin}-tag_on_failure>> |<<array,array>>|No
 | <<plugins-{type}s-{plugin}-target>> |<<string,string>>|No
@@ -143,6 +144,17 @@ For the built-in GeoLite2 City database, the following are available:
 `dma_code`, `ip`, `latitude`, `longitude`, `postal_code`, `region_code`,
 `region_name` and `timezone`.
 
+[id="plugins-{type}s-{plugin}-locales"]
+===== `locales`
+
+  * Value type is <<array,array>>
+  * There is no default value for this setting.
+
+An array of geoip locales to be included in the event.
+
+It displays the output of city_name or country_name in preferred locale. The one appear
+in the front of locales would resolve first, with 'en' as default.
+
 [id="plugins-{type}s-{plugin}-source"]
 ===== `source`
 

gradle/wrapper/gradle-wrapper.properties

@@ -1,6 +1,6 @@
-#Mon Oct 31 23:15:11 PDT 2016
+#Fri Dec 13 10:58:02 CST 2019
+distributionUrl=https\://services.gradle.org/distributions/gradle-3.1-all.zip
 distributionBase=GRADLE_USER_HOME
 distributionPath=wrapper/dists
-zipStoreBase=GRADLE_USER_HOME
 zipStorePath=wrapper/dists
-distributionUrl=https\://services.gradle.org/distributions/gradle-3.1-bin.zip
+zipStoreBase=GRADLE_USER_HOME

lib/logstash/filters/geoip.rb

@@ -56,6 +56,13 @@ class LogStash::Filters::GeoIP < LogStash::Filters::Base
   # `dma_code`, `ip`, `latitude`, `longitude`, `postal_code`, `region_name` and `timezone`.
   config :fields, :validate => :array
 
+  # An array of geoip locales to be used in the event.
+  #
+  # locales are gracefully resolved from beginning, with default as 'en'
+  #
+  # it matches locales support in mmdb files
+  config :locales, :validate => :array, :default => ['en']
+
   # Specify the field into which Logstash should store the geoip data.
   # This can be useful, for example, if you have `src_ip` and `dst_ip` fields and
   # would like the GeoIP information of both IPs.
@@ -99,8 +106,9 @@ def register
     end
 
     @logger.info("Using geoip database", :path => @database)
-
-    @geoipfilter = org.logstash.filters.GeoIPFilter.new(@source, @target, @fields, @database, @cache_size)
+
+    locales = (@locales + ['en']).uniq
+    @geoipfilter = org.logstash.filters.GeoIPFilter.new(@source, @target, locales, @fields, @database, @cache_size)
   end # def register
 
   public

logstash-filter-geoip.gemspec

@@ -1,7 +1,7 @@
 Gem::Specification.new do |s|
 
   s.name            = 'logstash-filter-geoip'
-  s.version         = '6.0.3'
+  s.version         = '6.0.4'
   s.licenses        = ['Apache License (2.0)']
   s.summary         = "Adds geographical information about an IP address"
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"

spec/filters/geoip_spec.rb

@@ -93,8 +93,8 @@
 
       it "should set other subfields of 'target' properly" do
         expect(event.get("target").to_hash.keys.sort).to eq(["city_name", "ip", "region_name"])
-        expect(event.get("[target][city_name]")).to eq("Nashua")
-        expect(event.get("[target][region_name]")).to eq("New Hampshire")
+        expect(event.get("[target][city_name]")).to eq("Brighton")
+        expect(event.get("[target][region_name]")).to eq("Massachusetts")
       end
 
     end
@@ -248,7 +248,7 @@
           expect(event.get("geoip")).not_to be_empty
           expect(event.get("geoip")["ip"]).to eq("2607:f0d0:1002:51:0:0:0:4")
           expect(event.get("geoip").to_hash.keys.sort).to eq(
-              ["continent_code", "country_code2", "country_code3", "country_name", "ip", "latitude", "location", "longitude", "timezone"]
+              ["city_name", "continent_code", "country_code2", "country_code3", "country_name", "dma_code", "ip", "latitude", "location", "longitude", "postal_code", "region_code", "region_name", "timezone"]
                                                           )
         end
       end
@@ -290,8 +290,31 @@
       expect(subject.get("geoip")["asn"]).to eq(15169)
       expect(subject.get("geoip")["as_org"]).to eq("Google Inc.")
     end
+  end
+
+  describe "support locales" do
+    config <<-CONFIG
+      filter {
+        geoip {
+          source => "ip"
+          locales => ["zh-CN", "pt-BR"]
+        }
+      }
+    CONFIG
 
+    sample("ip" => "106.38.102.106") do
+      expect(subject.get("geoip")).not_to be_empty
+      expect(subject.get("geoip")["ip"]).to eq("106.38.102.106")
+      expect(subject.get("geoip")["city_name"]).to eq("北京")
+      expect(subject.get("geoip")["country_name"]).to eq("中国")
+    end
 
+    sample("ip" => "189.2.0.0") do
+      expect(subject.get("geoip")).not_to be_empty
+      expect(subject.get("geoip")["ip"]).to eq("189.2.0.0")
+      expect(subject.get("geoip")["city_name"]).to eq("São Paulo")
+      expect(subject.get("geoip")["country_name"]).to eq("巴西")
+    end
   end
 
 end

src/main/java/org/logstash/filters/GeoIPFilter.java

@@ -59,12 +59,14 @@ public class GeoIPFilter {
 
   private final String sourceField;
   private final String targetField;
+  private final List<String> desiredLocales;
   private final Set<Fields> desiredFields;
   private final DatabaseReader databaseReader;
 
-  public GeoIPFilter(String sourceField, String targetField, List<String> fields, String databasePath, int cacheSize) {
+  public GeoIPFilter(String sourceField, String targetField, List<String> locales, List<String> fields, String databasePath, int cacheSize) {
     this.sourceField = sourceField;
     this.targetField = targetField;
+    this.desiredLocales = locales;
     final File database = new File(databasePath);
     try {
       this.databaseReader = new DatabaseReader.Builder(database).withCache(new CHMCache(cacheSize)).build();
@@ -206,7 +208,7 @@ private Map<String,Object> retrieveCityGeoData(InetAddress ipAddress) throws Geo
     for (Fields desiredField : this.desiredFields) {
       switch (desiredField) {
         case CITY_NAME:
-          String cityName = city.getName();
+          String cityName = getNameAccordingLocales(city.getNames(), this.desiredLocales);
           if (cityName != null) {
             geoData.put(Fields.CITY_NAME.fieldName(), cityName);
           }
@@ -218,13 +220,13 @@ private Map<String,Object> retrieveCityGeoData(InetAddress ipAddress) throws Geo
           }
           break;
         case CONTINENT_NAME:
-          String continentName = continent.getName();
+          String continentName = getNameAccordingLocales(continent.getNames(), this.desiredLocales);
           if (continentName != null) {
             geoData.put(Fields.CONTINENT_NAME.fieldName(), continentName);
           }
           break;
         case COUNTRY_NAME:
-          String countryName = country.getName();
+          String countryName = getNameAccordingLocales(country.getNames(), this.desiredLocales);
           if (countryName != null) {
             geoData.put(Fields.COUNTRY_NAME.fieldName(), countryName);
           }
@@ -257,7 +259,7 @@ private Map<String,Object> retrieveCityGeoData(InetAddress ipAddress) throws Geo
           }
           break;
         case REGION_NAME:
-          String subdivisionName = subdivision.getName();
+          String subdivisionName = getNameAccordingLocales(subdivision.getNames(), this.desiredLocales);
           if (subdivisionName != null) {
             geoData.put(Fields.REGION_NAME.fieldName(), subdivisionName);
           }
@@ -320,13 +322,13 @@ private Map<String,Object> retrieveCountryGeoData(InetAddress ipAddress) throws
           }
           break;
         case COUNTRY_NAME:
-          String countryName = country.getName();
+          String countryName = getNameAccordingLocales(country.getNames(), this.desiredLocales);
           if (countryName != null) {
             geoData.put(Fields.COUNTRY_NAME.fieldName(), countryName);
           }
           break;
         case CONTINENT_NAME:
-          String continentName = continent.getName();
+          String continentName = getNameAccordingLocales(continent.getNames(), this.desiredLocales);
           if (continentName != null) {
             geoData.put(Fields.CONTINENT_NAME.fieldName(), continentName);
           }
@@ -401,4 +403,17 @@ private Map<String, Object> retrieveAsnGeoData(InetAddress ipAddress) throws Geo
 
     return geoData;
   }
+
+  private String getNameAccordingLocales(Map<String, String> names, List<String> locales) {
+    int size = locales.size();
+    if (size <= 0) {
+      return null;
+    }
+
+    String first = locales.get(0);
+    if (names.containsKey(first)) {
+      return names.get(first);
+    }
+    return this.getNameAccordingLocales(names, locales.subList(1, size));
+  }
 }

vendor.json

@@ -1,7 +1,7 @@
 [
     {
         "url": "http://geolite.maxmind.com/download/geoip/database/GeoLite2-City.mmdb.gz",
-        "sha1": "8632d2abf2672fb33aec3845d0dcbb82986d1b4a"
+        "sha1": "f3c9d9c213de73c9b89ab2a965e022b2d4ce38a7"
     },
     {
         "url": "https://s3.amazonaws.com/download.elasticsearch.org/logstash/maxmind/GeoLite2-ASN.mmdb",