Merge pull request #2 from nash-io/add_ex_rated

Add ex_rated

.github/workflows/elixir.yml

@@ -10,14 +10,16 @@ jobs:
   build:
 
     runs-on: ubuntu-latest
-
+    strategy:
+      matrix:
+        otp: ['24.1']
+        elixir: ['1.12.3']
     steps:
     - uses: actions/checkout@v2
-    - name: Setup elixir
-      uses: actions/setup-elixir@v1
+    - uses: erlef/setup-beam@v1
       with:
-        elixir-version: 1.10.3
-        otp-version: 22.2
+        otp-version: ${{matrix.otp}}
+        elixir-version: ${{matrix.elixir}}
     - name: deps cache
       uses: actions/cache@v1
       with:
@@ -32,7 +34,7 @@ jobs:
         key: ${{ runner.os }}-build-${{ hashFiles(format('{0}{1}', github.workspace, '/mix.lock')) }}
         restore-keys: |
           ${{ runner.os }}-build-
-    - name: dializer cache
+    - name: dialyzer cache
       uses: actions/cache@v1
       with:
         path: deps

.gitignore

@@ -23,3 +23,6 @@ erl_crash.dump
 http_client-*.tar
 
 .elixir_ls/
+
+*.plt
+*.xml

README.md

@@ -1,21 +1,31 @@
 # HttpClient
 
-**TODO: Add description**
+Httpoison boosted with telemetry, mox and ex_rated.
+
+```elixir
+HttpClient.get("http://mydomain.com")
+```
 
 ## Installation
 
-If [available in Hex](https://hex.pm/docs/publish), the package can be installed
-by adding `http_client` to your list of dependencies in `mix.exs`:
+The package can be installed by adding `http_client` to your list of dependencies 
+in `mix.exs`:
 
 ```elixir
 def deps do
   [
-    {:http_client, "~> 0.1.0"}
+    {:http_client, "~> 0.2.3"}
   ]
 end
 ```
 
-Documentation can be generated with [ExDoc](https://github.com/elixir-lang/ex_doc)
-and published on [HexDocs](https://hexdocs.pm). Once published, the docs can
-be found at [https://hexdocs.pm/http_client](https://hexdocs.pm/http_client).
+The docs can be found at [https://hexdocs.pm/http_client](https://hexdocs.pm/http_client).
+
+## Configuration
+
+```elixir
+# Will rate limit calls to mydomain.com to 5 per seconds
+# HttpClient.get("http://mydomain.com/any_path")
+config :http_client, :rate_limits, "mydomain.com": [{:timer.seconds(1), 5}]
+```
 

lib/http_client.ex

@@ -1,6 +1,8 @@
 defmodule HttpClient do
   @moduledoc false
   alias HttpClient.Instrumenter
+  alias HttpClient.RateLimiter
+
   @callback get(String.t()) :: {:ok, any()} | {:error, any()}
   @callback get(String.t(), list()) :: {:ok, any()} | {:error, any()}
   @callback get(String.t(), list(), Keyword.t()) :: {:ok, any()} | {:error, any()}
@@ -17,87 +19,35 @@ defmodule HttpClient do
   @callback delete(String.t(), list()) :: {:ok, any()} | {:error, any()}
   @callback delete(String.t(), list(), Keyword.t()) :: {:ok, any()} | {:error, any()}
 
-  def get(url) do
-    :timer.tc(fn ->
-      impl().get(url)
-    end)
-    |> Instrumenter.instrument(url)
-  end
+  def get(url), do: request(:get, [url])
 
-  def get(url, headers) do
-    :timer.tc(fn ->
-      impl().get(url, headers)
-    end)
-    |> Instrumenter.instrument(url)
-  end
+  def get(url, headers), do: request(:get, [url, headers])
 
-  def get(url, headers, opts) do
-    :timer.tc(fn ->
-      impl().get(url, headers, opts)
-    end)
-    |> Instrumenter.instrument(url)
-  end
+  def get(url, headers, options), do: request(:get, [url, headers, options])
 
-  def post(url, body) do
-    :timer.tc(fn ->
-      impl().post(url, body)
-    end)
-    |> Instrumenter.instrument(url)
-  end
+  def post(url, body), do: request(:post, [url, body])
 
-  def post(url, body, headers) do
-    :timer.tc(fn ->
-      impl().post(url, body, headers)
-    end)
-    |> Instrumenter.instrument(url)
-  end
+  def post(url, body, headers), do: request(:post, [url, body, headers])
 
-  def post(url, body, headers, opts) do
-    :timer.tc(fn ->
-      impl().post(url, body, headers, opts)
-    end)
-    |> Instrumenter.instrument(url)
-  end
+  def post(url, body, headers, options), do: request(:post, [url, body, headers, options])
 
-  def put(url, body) do
-    :timer.tc(fn ->
-      impl().put(url, body)
-    end)
-    |> Instrumenter.instrument(url)
-  end
+  def put(url, body), do: request(:put, [url, body])
 
-  def put(url, body, headers) do
-    :timer.tc(fn ->
-      impl().put(url, body, headers)
-    end)
-    |> Instrumenter.instrument(url)
-  end
+  def put(url, body, headers), do: request(:put, [url, body, headers])
 
-  def put(url, body, headers, opts) do
-    :timer.tc(fn ->
-      impl().put(url, body, headers, opts)
-    end)
-    |> Instrumenter.instrument(url)
-  end
+  def put(url, body, headers, options), do: request(:put, [url, body, headers, options])
 
-  def delete(url) do
-    :timer.tc(fn ->
-      impl().delete(url)
-    end)
-    |> Instrumenter.instrument(url)
-  end
+  def delete(url), do: request(:delete, [url])
 
-  def delete(url, headers) do
-    :timer.tc(fn ->
-      impl().delete(url, headers)
-    end)
-    |> Instrumenter.instrument(url)
-  end
+  def delete(url, headers), do: request(:delete, [url, headers])
+
+  def delete(url, headers, options), do: request(:delete, [url, headers, options])
+
+  defp request(method, [url | _] = args) do
+    :ok = RateLimiter.rate_limit(url)
 
-  def delete(url, headers, options) do
-    :timer.tc(fn ->
-      impl().delete(url, headers, options)
-    end)
+    fn -> apply(impl(), method, args) end
+    |> :timer.tc()
     |> Instrumenter.instrument(url)
   end
 

lib/http_client/rate_limiter.ex

@@ -0,0 +1,50 @@
+defmodule HttpClient.RateLimiter do
+  @moduledoc """
+  Module used to rate limit calls to certain host
+
+  In `config.exs` :
+
+  config :http_client, :rate_limits,
+    "google.com": [{:timer.seconds(1), 5}, {:timer.hours(24), 500_000}]
+  """
+
+  @spec rate_limit(String.t()) :: :ok
+  def rate_limit(url) do
+    case retrieve_rate_limit_config(url) do
+      {:ok, {host, rate_limits}} ->
+        :ok = enforce_rate_limits(host, rate_limits)
+
+      :error ->
+        :ok
+    end
+  end
+
+  @spec enforce_rate_limits(atom(), list({non_neg_integer(), non_neg_integer()})) :: :ok
+  defp enforce_rate_limits(_host, []), do: :ok
+
+  defp enforce_rate_limits(host, [{scale, limit} | remaining_limits] = rate_limits) do
+    bucket = {host, scale}
+    {_, _, ms_to_next_bucket, _, _} = ExRated.inspect_bucket(bucket, scale, limit)
+
+    case ExRated.check_rate(bucket, scale, limit) do
+      {:ok, _} ->
+        enforce_rate_limits(host, remaining_limits)
+
+      _ ->
+        Process.sleep(ms_to_next_bucket)
+        enforce_rate_limits(host, rate_limits)
+    end
+  end
+
+  @spec retrieve_rate_limit_config(String.t()) ::
+          {:ok, {atom(), list({non_neg_integer(), non_neg_integer()})}} | :error
+  defp retrieve_rate_limit_config(url) do
+    %URI{host: host} = URI.parse(url)
+    host_atom = String.to_existing_atom(host)
+    {:ok, rate_limits} = Application.fetch_env(:http_client, :rate_limits)
+    {:ok, value} = Keyword.fetch(rate_limits, host_atom)
+    {:ok, {host_atom, value}}
+  rescue
+    _ -> :error
+  end
+end

mix.exs

@@ -4,7 +4,7 @@ defmodule HttpClient.MixProject do
   def project do
     [
       app: :http_client,
-      version: "0.2.2",
+      version: "0.2.3",
       elixir: "~> 1.10",
       start_permanent: Mix.env() == :prod,
       test_coverage: [tool: ExCoveralls],
@@ -20,18 +20,15 @@ defmodule HttpClient.MixProject do
     ]
   end
 
-  # Run "mix help compile.app" to learn about applications.
   def application do
-    [
-      extra_applications: [:logger]
-    ]
+    [extra_applications: [:logger]]
   end
 
-  # Run "mix help deps" to learn about dependencies.
   defp deps do
     [
       {:httpoison, "~> 1.5"},
       {:telemetry, "~> 0.4.0"},
+      {:ex_rated, "~> 2.0"},
       {:mox, "~> 0.5", only: :test},
       {:excoveralls, "~> 0.12", only: [:test]},
       {:ex_unit_sonarqube, "~> 0.1.2", only: [:dev, :test]},
@@ -44,7 +41,7 @@ defmodule HttpClient.MixProject do
   end
 
   defp description do
-    "Httpoison boosted with telemetry and mox."
+    "Httpoison boosted with telemetry, mox and ex_rated."
   end
 
   defp package do