WIP: update dependencies and remove use of logger_file_backend

lib/crawly/data_storage/data_storage.ex

@@ -58,7 +58,7 @@ defmodule Crawly.DataStorage do
     {:ok, %Crawly.DataStorage{workers: %{}, pid_spiders: %{}}}
   end
 
-  def handle_call({:store, spider, item}, _from, state) do
+  def handle_call({:store, spider, item}, _from, %Crawly.DataStorage{} = state) do
     %{workers: workers} = state
 
     message =
@@ -73,7 +73,11 @@ defmodule Crawly.DataStorage do
     {:reply, message, state}
   end
 
-  def handle_call({:start_worker, spider_name, crawl_id}, _from, state) do
+  def handle_call(
+        {:start_worker, spider_name, crawl_id},
+        _from,
+        %Crawly.DataStorage{} = state
+      ) do
     {msg, new_state} =
       case Map.get(state.workers, spider_name) do
         nil ->

lib/crawly/data_storage/data_storage_worker.ex

@@ -52,7 +52,7 @@ defmodule Crawly.DataStorage.Worker do
         {false, new_state} ->
           new_state
 
-        {new_item, new_state} ->
+        {new_item, %Worker{} = new_state} ->
           Logger.debug("Stored item: #{inspect(new_item)}")
           %Worker{new_state | stored_items: state.stored_items + 1}
       end

lib/crawly/engine.ex

@@ -61,18 +61,8 @@ defmodule Crawly.Engine do
       |> Map.put_new_lazy(:crawl_id, &UUID.uuid1/0)
 
     # Filter all logs related to a given spider
-    case {Crawly.Utils.get_settings(:log_to_file, spider_name),
-          Crawly.Utils.ensure_loaded?(LoggerFileBackend)} do
-      {true, true} ->
-        configure_spider_logs(spider_name, opts[:crawl_id])
-
-      {true, false} ->
-        Logger.error(
-          ":logger_file_backend https://github.com/onkel-dirtus/logger_file_backend#loggerfilebackend must be installed as a peer dependency if log_to_file config is set to true"
-        )
-
-      _ ->
-        false
+    if Crawly.Utils.get_settings(:log_to_file, spider_name) do
+      configure_spider_logs(spider_name, opts[:crawl_id])
     end
 
     GenServer.call(
@@ -186,7 +176,7 @@ defmodule Crawly.Engine do
   def handle_call(
         {:start_spider, spider_name, crawl_id, options},
         _form,
-        state
+        %Crawly.Engine{} = state
       ) do
     result =
       case Map.get(state.started_spiders, spider_name) do
@@ -212,7 +202,11 @@ defmodule Crawly.Engine do
     {:reply, msg, %Crawly.Engine{state | started_spiders: new_started_spiders}}
   end
 
-  def handle_call({:stop_spider, spider_name, reason}, _form, state) do
+  def handle_call(
+        {:stop_spider, spider_name, reason},
+        _form,
+        %Crawly.Engine{} = state
+      ) do
     {msg, new_started_spiders} =
       case Map.pop(state.started_spiders, spider_name) do
         {nil, _} ->
@@ -233,13 +227,15 @@ defmodule Crawly.Engine do
 
           Crawly.EngineSup.stop_spider(pid)
 
+          remove_spider_logs(spider_name, crawl_id)
+
           {:ok, new_started_spiders}
       end
 
     {:reply, msg, %Crawly.Engine{state | started_spiders: new_started_spiders}}
   end
 
-  def handle_call({:get_spider, spider_name}, _from, state) do
+  def handle_call({:get_spider, spider_name}, _from, %Crawly.Engine{} = state) do
     return =
       if Enum.member?(state.known_spiders, spider_name) do
         format_spider_info(spider_name, state)
@@ -248,7 +244,7 @@ defmodule Crawly.Engine do
     {:reply, return, state}
   end
 
-  def handle_cast(:refresh_spider_list, state) do
+  def handle_cast(:refresh_spider_list, %Crawly.Engine{} = state) do
     updated = get_updated_known_spider_list(state.known_spiders)
     {:noreply, %Crawly.Engine{state | known_spiders: updated}}
   end
@@ -273,14 +269,64 @@ defmodule Crawly.Engine do
 
   defp configure_spider_logs(spider_name, crawl_id) do
     log_file_path = Crawly.Utils.spider_log_path(spider_name, crawl_id)
-    Logger.add_backend({LoggerFileBackend, :debug})
 
-    Logger.configure_backend({LoggerFileBackend, :debug},
-      path: Crawly.Utils.spider_log_path(spider_name, crawl_id),
+    log_file_path
+    |> Path.dirname()
+    |> File.mkdir_p!()
+
+    handler_id = spider_handler_id(spider_name, crawl_id)
+
+    filter = {
+      fn log_event, _extra ->
+        case log_event do
+          %{meta: %{crawl_id: ^crawl_id}} -> log_event
+          _ -> :stop
+        end
+      end,
+      []
+    }
+
+    config = %{
       level: :debug,
-      metadata_filter: [crawl_id: crawl_id]
-    )
+      formatter:
+        {:logger_formatter,
+         %{template: [:time, " [", :level, "] ", :msg, "\n"]}},
+      filters: [spider_filter: filter],
+      config: %{
+        file: String.to_charlist(log_file_path),
+        filesync_repeat_interval: :no_repeat
+      }
+    }
+
+    handler_id
+    |> :logger.add_handler(:logger_std_h, config)
+    |> case do
+      :ok ->
+        Logger.metadata(crawl_id: crawl_id)
+        Logger.debug("Writing logs to #{log_file_path}")
+
+      {:error, {:already_exist, _}} ->
+        :ok
+
+      {:error, error} ->
+        Logger.error(
+          "Failed to configure log handler #{inspect(handler_id)}: #{inspect(error)}"
+        )
+    end
+  end
+
+  defp remove_spider_logs(spider_name, crawl_id) do
+    handler_id = spider_handler_id(spider_name, crawl_id)
+    :logger.remove_handler(handler_id)
+  end
+
+  defp spider_handler_id(spider_name, crawl_id) do
+    spider_name_str =
+      case Atom.to_string(spider_name) do
+        "Elixir." <> name_str -> name_str
+        name_str -> name_str
+      end
 
-    Logger.debug("Writing logs to #{log_file_path}")
+    String.to_atom("crawly_#{spider_name_str}_#{crawl_id}")
   end
 end

lib/crawly/fetchers/httpoison_fetcher.ex

@@ -4,8 +4,6 @@ defmodule Crawly.Fetchers.HTTPoisonFetcher do
   """
   @behaviour Crawly.Fetchers.Fetcher
 
-  require Logger
-
   def fetch(request, _client_options) do
     # TODO: This should return Crawly.Response.
     HTTPoison.get(request.url, request.headers, request.options)

lib/crawly/fetchers/splash.ex

@@ -21,16 +21,14 @@ defmodule Crawly.Fetchers.Splash do
   """
   @behaviour Crawly.Fetchers.Fetcher
 
-  require Logger
-
   @spec fetch(request, client_options) :: response
         when request: Crawly.Request.t(),
              client_options: [binary()],
              response: Crawly.Response.t()
   def fetch(request, client_options) do
     {base_url, other_options} =
       case Keyword.pop(client_options, :base_url, nil) do
-        nil ->
+        {nil, _opts} ->
           throw(
             "The base_url is not set. Splash fetcher can't be used! " <>
               "Please set :base_url in fetcher options to continue. " <>
@@ -50,8 +48,9 @@ defmodule Crawly.Fetchers.Splash do
       |> URI.to_string()
 
     case HTTPoison.get(url, request.headers, request.options) do
-      {:ok, response} ->
-        new_request = %HTTPoison.Request{response.request | url: request.url}
+      {:ok,
+       %HTTPoison.Response{request: %HTTPoison.Request{} = request} = response} ->
+        new_request = %HTTPoison.Request{request | url: request.url}
 
         new_response = %HTTPoison.Response{
           response

lib/crawly/middlewares/request_options.ex

@@ -24,7 +24,7 @@ defmodule Crawly.Middlewares.RequestOptions do
   """
   @behaviour Crawly.Pipeline
 
-  def run(request, state, options \\ []) do
+  def run(%Crawly.Request{} = request, state, options \\ []) do
     {%Crawly.Request{
        request
        | options: Keyword.merge(options, request.options)

lib/crawly/models/job.ex

@@ -93,7 +93,7 @@ defmodule Crawly.Models.Job do
   @spec update(String.t(), integer(), term()) :: :ok | {:error, any}
   def update(crawl_id, total_scraped, stop_reason) do
     case Crawly.SimpleStorage.get(@table_name, crawl_id) do
-      {:ok, job_item} ->
+      {:ok, %Crawly.Models.Job{} = job_item} ->
         new_job_item = %Crawly.Models.Job{
           job_item
           | stop_reason: stop_reason,

lib/crawly/worker.ex

@@ -176,7 +176,7 @@ defmodule Crawly.Worker do
   end
 
   ## Retry a request if max retries allows to do so
-  defp maybe_retry_request(spider, request) do
+  defp maybe_retry_request(spider, %Crawly.Request{} = request) do
     retries = request.retries
     retry_settings = Crawly.Utils.get_settings(:retry, spider, Keyword.new())
 

mix.exs

@@ -45,22 +45,21 @@ defmodule Crawly.Mixfile do
   # Run "mix help deps" to learn about dependencies.
   defp deps do
     [
-      {:httpoison, "~> 2.2"},
+      {:httpoison, "~> 3.0"},
       {:elixir_uuid, "~> 1.2"},
-      {:poison, "~> 3.1"},
-      {:gollum, "~> 0.5.0", hex: :new_gollum},
-      {:plug_cowboy, "~> 2.0"},
+      {:poison, "~> 6.0.0"},
+      {:gollum, "~> 0.5.0",
+       github: "cmeon/gollum", branch: "update-dependencies"},
+      {:plug_cowboy, "~> 2.9.0"},
       {:credo, "~> 1.7", only: [:dev, :test], runtime: false},
       {:ex_doc, ">= 0.0.0", only: :dev, runtime: false},
-      {:earmark, "~> 1.2", only: :dev},
-      {:meck, "~> 0.9", only: :test},
-      {:excoveralls, "~> 0.14.6", only: :test},
-      {:yaml_elixir, "~> 2.9"},
-      {:ex_json_schema, "~> 0.9.2"},
+      {:mdex, "~> 0.13.5", only: :dev},
+      {:meck, "~> 1.2.0", only: :test},
+      {:excoveralls, "~> 0.18.5", only: :test},
+      {:yaml_elixir, "~> 2.12.2"},
+      {:ex_json_schema, "~> 0.11.3"},
       # Add floki only for crawly standalone release
-      {:floki, "~> 0.33.0", only: [:dev, :test, :standalone_crawly]},
-      {:logger_file_backend, "~> 0.0.11",
-       only: [:test, :dev, :standalone_crawly]}
+      {:floki, "~> 0.38.4", only: [:dev, :test, :standalone_crawly]}
     ]
   end