diff --git a/lib/crawly/data_storage/data_storage.ex b/lib/crawly/data_storage/data_storage.ex index cf0f7785..8d5dfc11 100644 --- a/lib/crawly/data_storage/data_storage.ex +++ b/lib/crawly/data_storage/data_storage.ex @@ -58,7 +58,7 @@ defmodule Crawly.DataStorage do {:ok, %Crawly.DataStorage{workers: %{}, pid_spiders: %{}}} end - def handle_call({:store, spider, item}, _from, state) do + def handle_call({:store, spider, item}, _from, %Crawly.DataStorage{} = state) do %{workers: workers} = state message = @@ -73,7 +73,11 @@ defmodule Crawly.DataStorage do {:reply, message, state} end - def handle_call({:start_worker, spider_name, crawl_id}, _from, state) do + def handle_call( + {:start_worker, spider_name, crawl_id}, + _from, + %Crawly.DataStorage{} = state + ) do {msg, new_state} = case Map.get(state.workers, spider_name) do nil -> diff --git a/lib/crawly/data_storage/data_storage_worker.ex b/lib/crawly/data_storage/data_storage_worker.ex index c3426c1f..e6c69bc7 100644 --- a/lib/crawly/data_storage/data_storage_worker.ex +++ b/lib/crawly/data_storage/data_storage_worker.ex @@ -52,7 +52,7 @@ defmodule Crawly.DataStorage.Worker do {false, new_state} -> new_state - {new_item, new_state} -> + {new_item, %Worker{} = new_state} -> Logger.debug("Stored item: #{inspect(new_item)}") %Worker{new_state | stored_items: state.stored_items + 1} end diff --git a/lib/crawly/engine.ex b/lib/crawly/engine.ex index 354e489b..eb7117e9 100644 --- a/lib/crawly/engine.ex +++ b/lib/crawly/engine.ex @@ -61,18 +61,8 @@ defmodule Crawly.Engine do |> Map.put_new_lazy(:crawl_id, &UUID.uuid1/0) # Filter all logs related to a given spider - case {Crawly.Utils.get_settings(:log_to_file, spider_name), - Crawly.Utils.ensure_loaded?(LoggerFileBackend)} do - {true, true} -> - configure_spider_logs(spider_name, opts[:crawl_id]) - - {true, false} -> - Logger.error( - ":logger_file_backend https://github.com/onkel-dirtus/logger_file_backend#loggerfilebackend must be installed as a peer dependency if log_to_file config is set to true" - ) - - _ -> - false + if Crawly.Utils.get_settings(:log_to_file, spider_name) do + configure_spider_logs(spider_name, opts[:crawl_id]) end GenServer.call( @@ -186,7 +176,7 @@ defmodule Crawly.Engine do def handle_call( {:start_spider, spider_name, crawl_id, options}, _form, - state + %Crawly.Engine{} = state ) do result = case Map.get(state.started_spiders, spider_name) do @@ -212,7 +202,11 @@ defmodule Crawly.Engine do {:reply, msg, %Crawly.Engine{state | started_spiders: new_started_spiders}} end - def handle_call({:stop_spider, spider_name, reason}, _form, state) do + def handle_call( + {:stop_spider, spider_name, reason}, + _form, + %Crawly.Engine{} = state + ) do {msg, new_started_spiders} = case Map.pop(state.started_spiders, spider_name) do {nil, _} -> @@ -233,13 +227,15 @@ defmodule Crawly.Engine do Crawly.EngineSup.stop_spider(pid) + remove_spider_logs(spider_name, crawl_id) + {:ok, new_started_spiders} end {:reply, msg, %Crawly.Engine{state | started_spiders: new_started_spiders}} end - def handle_call({:get_spider, spider_name}, _from, state) do + def handle_call({:get_spider, spider_name}, _from, %Crawly.Engine{} = state) do return = if Enum.member?(state.known_spiders, spider_name) do format_spider_info(spider_name, state) @@ -248,7 +244,7 @@ defmodule Crawly.Engine do {:reply, return, state} end - def handle_cast(:refresh_spider_list, state) do + def handle_cast(:refresh_spider_list, %Crawly.Engine{} = state) do updated = get_updated_known_spider_list(state.known_spiders) {:noreply, %Crawly.Engine{state | known_spiders: updated}} end @@ -273,14 +269,64 @@ defmodule Crawly.Engine do defp configure_spider_logs(spider_name, crawl_id) do log_file_path = Crawly.Utils.spider_log_path(spider_name, crawl_id) - Logger.add_backend({LoggerFileBackend, :debug}) - Logger.configure_backend({LoggerFileBackend, :debug}, - path: Crawly.Utils.spider_log_path(spider_name, crawl_id), + log_file_path + |> Path.dirname() + |> File.mkdir_p!() + + handler_id = spider_handler_id(spider_name, crawl_id) + + filter = { + fn log_event, _extra -> + case log_event do + %{meta: %{crawl_id: ^crawl_id}} -> log_event + _ -> :stop + end + end, + [] + } + + config = %{ level: :debug, - metadata_filter: [crawl_id: crawl_id] - ) + formatter: + {:logger_formatter, + %{template: [:time, " [", :level, "] ", :msg, "\n"]}}, + filters: [spider_filter: filter], + config: %{ + file: String.to_charlist(log_file_path), + filesync_repeat_interval: :no_repeat + } + } + + handler_id + |> :logger.add_handler(:logger_std_h, config) + |> case do + :ok -> + Logger.metadata(crawl_id: crawl_id) + Logger.debug("Writing logs to #{log_file_path}") + + {:error, {:already_exist, _}} -> + :ok + + {:error, error} -> + Logger.error( + "Failed to configure log handler #{inspect(handler_id)}: #{inspect(error)}" + ) + end + end + + defp remove_spider_logs(spider_name, crawl_id) do + handler_id = spider_handler_id(spider_name, crawl_id) + :logger.remove_handler(handler_id) + end + + defp spider_handler_id(spider_name, crawl_id) do + spider_name_str = + case Atom.to_string(spider_name) do + "Elixir." <> name_str -> name_str + name_str -> name_str + end - Logger.debug("Writing logs to #{log_file_path}") + String.to_atom("crawly_#{spider_name_str}_#{crawl_id}") end end diff --git a/lib/crawly/fetchers/httpoison_fetcher.ex b/lib/crawly/fetchers/httpoison_fetcher.ex index 48b735db..f6e2cc18 100644 --- a/lib/crawly/fetchers/httpoison_fetcher.ex +++ b/lib/crawly/fetchers/httpoison_fetcher.ex @@ -4,8 +4,6 @@ defmodule Crawly.Fetchers.HTTPoisonFetcher do """ @behaviour Crawly.Fetchers.Fetcher - require Logger - def fetch(request, _client_options) do # TODO: This should return Crawly.Response. HTTPoison.get(request.url, request.headers, request.options) diff --git a/lib/crawly/fetchers/splash.ex b/lib/crawly/fetchers/splash.ex index 2875ea12..0fb1ec2a 100644 --- a/lib/crawly/fetchers/splash.ex +++ b/lib/crawly/fetchers/splash.ex @@ -21,8 +21,6 @@ defmodule Crawly.Fetchers.Splash do """ @behaviour Crawly.Fetchers.Fetcher - require Logger - @spec fetch(request, client_options) :: response when request: Crawly.Request.t(), client_options: [binary()], @@ -30,7 +28,7 @@ defmodule Crawly.Fetchers.Splash do def fetch(request, client_options) do {base_url, other_options} = case Keyword.pop(client_options, :base_url, nil) do - nil -> + {nil, _opts} -> throw( "The base_url is not set. Splash fetcher can't be used! " <> "Please set :base_url in fetcher options to continue. " <> @@ -50,8 +48,9 @@ defmodule Crawly.Fetchers.Splash do |> URI.to_string() case HTTPoison.get(url, request.headers, request.options) do - {:ok, response} -> - new_request = %HTTPoison.Request{response.request | url: request.url} + {:ok, + %HTTPoison.Response{request: %HTTPoison.Request{} = request} = response} -> + new_request = %HTTPoison.Request{request | url: request.url} new_response = %HTTPoison.Response{ response diff --git a/lib/crawly/middlewares/request_options.ex b/lib/crawly/middlewares/request_options.ex index d96fb1a2..dbdcb088 100644 --- a/lib/crawly/middlewares/request_options.ex +++ b/lib/crawly/middlewares/request_options.ex @@ -24,7 +24,7 @@ defmodule Crawly.Middlewares.RequestOptions do """ @behaviour Crawly.Pipeline - def run(request, state, options \\ []) do + def run(%Crawly.Request{} = request, state, options \\ []) do {%Crawly.Request{ request | options: Keyword.merge(options, request.options) diff --git a/lib/crawly/models/job.ex b/lib/crawly/models/job.ex index 2bbf3508..b10d1377 100644 --- a/lib/crawly/models/job.ex +++ b/lib/crawly/models/job.ex @@ -93,7 +93,7 @@ defmodule Crawly.Models.Job do @spec update(String.t(), integer(), term()) :: :ok | {:error, any} def update(crawl_id, total_scraped, stop_reason) do case Crawly.SimpleStorage.get(@table_name, crawl_id) do - {:ok, job_item} -> + {:ok, %Crawly.Models.Job{} = job_item} -> new_job_item = %Crawly.Models.Job{ job_item | stop_reason: stop_reason, diff --git a/lib/crawly/worker.ex b/lib/crawly/worker.ex index 35ecdde7..9ced129d 100644 --- a/lib/crawly/worker.ex +++ b/lib/crawly/worker.ex @@ -176,7 +176,7 @@ defmodule Crawly.Worker do end ## Retry a request if max retries allows to do so - defp maybe_retry_request(spider, request) do + defp maybe_retry_request(spider, %Crawly.Request{} = request) do retries = request.retries retry_settings = Crawly.Utils.get_settings(:retry, spider, Keyword.new()) diff --git a/mix.exs b/mix.exs index 8ddcf862..f405a4ca 100644 --- a/mix.exs +++ b/mix.exs @@ -45,22 +45,21 @@ defmodule Crawly.Mixfile do # Run "mix help deps" to learn about dependencies. defp deps do [ - {:httpoison, "~> 2.2"}, + {:httpoison, "~> 3.0"}, {:elixir_uuid, "~> 1.2"}, - {:poison, "~> 3.1"}, - {:gollum, "~> 0.5.0", hex: :new_gollum}, - {:plug_cowboy, "~> 2.0"}, + {:poison, "~> 6.0.0"}, + {:gollum, "~> 0.5.0", + github: "cmeon/gollum", branch: "update-dependencies"}, + {:plug_cowboy, "~> 2.9.0"}, {:credo, "~> 1.7", only: [:dev, :test], runtime: false}, {:ex_doc, ">= 0.0.0", only: :dev, runtime: false}, - {:earmark, "~> 1.2", only: :dev}, - {:meck, "~> 0.9", only: :test}, - {:excoveralls, "~> 0.14.6", only: :test}, - {:yaml_elixir, "~> 2.9"}, - {:ex_json_schema, "~> 0.9.2"}, + {:mdex, "~> 0.13.5", only: :dev}, + {:meck, "~> 1.2.0", only: :test}, + {:excoveralls, "~> 0.18.5", only: :test}, + {:yaml_elixir, "~> 2.12.2"}, + {:ex_json_schema, "~> 0.11.3"}, # Add floki only for crawly standalone release - {:floki, "~> 0.33.0", only: [:dev, :test, :standalone_crawly]}, - {:logger_file_backend, "~> 0.0.11", - only: [:test, :dev, :standalone_crawly]} + {:floki, "~> 0.38.4", only: [:dev, :test, :standalone_crawly]} ] end diff --git a/mix.lock b/mix.lock index 7ce2e1b1..794efe7b 100644 --- a/mix.lock +++ b/mix.lock @@ -1,49 +1,46 @@ %{ "bunt": {:hex, :bunt, "1.0.0", "081c2c665f086849e6d57900292b3a161727ab40431219529f13c4ddcf3e7a44", [:mix], [], "hexpm", "dc5f86aa08a5f6fa6b8096f0735c4e76d54ae5c9fa2c143e5a1fc7c1cd9bb6b5"}, - "castore": {:hex, :castore, "1.0.1", "240b9edb4e9e94f8f56ab39d8d2d0a57f49e46c56aced8f873892df8ff64ff5a", [:mix], [], "hexpm", "b4951de93c224d44fac71614beabd88b71932d0b1dea80d2f80fb9044e01bbb3"}, - "certifi": {:hex, :certifi, "2.12.0", "2d1cca2ec95f59643862af91f001478c9863c2ac9cb6e2f89780bfd8de987329", [:rebar3], [], "hexpm", "ee68d85df22e554040cdb4be100f33873ac6051387baf6a8f6ce82272340ff1c"}, - "cowboy": {:hex, :cowboy, "2.9.0", "865dd8b6607e14cf03282e10e934023a1bd8be6f6bacf921a7e2a96d800cd452", [:make, :rebar3], [{:cowlib, "2.11.0", [hex: :cowlib, repo: "hexpm", optional: false]}, {:ranch, "1.8.0", [hex: :ranch, repo: "hexpm", optional: false]}], "hexpm", "2c729f934b4e1aa149aff882f57c6372c15399a20d54f65c8d67bef583021bde"}, - "cowboy_telemetry": {:hex, :cowboy_telemetry, "0.3.1", "ebd1a1d7aff97f27c66654e78ece187abdc646992714164380d8a041eda16754", [:rebar3], [{:cowboy, "~> 2.7", [hex: :cowboy, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "3a6efd3366130eab84ca372cbd4a7d3c3a97bdfcfb4911233b035d117063f0af"}, - "cowlib": {:hex, :cowlib, "2.11.0", "0b9ff9c346629256c42ebe1eeb769a83c6cb771a6ee5960bd110ab0b9b872063", [:make, :rebar3], [], "hexpm", "2b3e9da0b21c4565751a6d4901c20d1b4cc25cbb7fd50d91d2ab6dd287bc86a9"}, - "credo": {:hex, :credo, "1.7.7", "771445037228f763f9b2afd612b6aa2fd8e28432a95dbbc60d8e03ce71ba4446", [:mix], [{:bunt, "~> 0.2.1 or ~> 1.0", [hex: :bunt, repo: "hexpm", optional: false]}, {:file_system, "~> 0.2 or ~> 1.0", [hex: :file_system, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}], "hexpm", "8bc87496c9aaacdc3f90f01b7b0582467b69b4bd2441fe8aae3109d843cc2f2e"}, - "decimal": {:hex, :decimal, "2.1.1", "5611dca5d4b2c3dd497dec8f68751f1f1a54755e8ed2a966c2633cf885973ad6", [:mix], [], "hexpm", "53cfe5f497ed0e7771ae1a475575603d77425099ba5faef9394932b35020ffcc"}, - "earmark": {:hex, :earmark, "1.4.15", "2c7f924bf495ec1f65bd144b355d0949a05a254d0ec561740308a54946a67888", [:mix], [{:earmark_parser, ">= 1.4.13", [hex: :earmark_parser, repo: "hexpm", optional: false]}], "hexpm", "3b1209b85bc9f3586f370f7c363f6533788fb4e51db23aa79565875e7f9999ee"}, - "earmark_parser": {:hex, :earmark_parser, "1.4.41", "ab34711c9dc6212dda44fcd20ecb87ac3f3fce6f0ca2f28d4a00e4154f8cd599", [:mix], [], "hexpm", "a81a04c7e34b6617c2792e291b5a2e57ab316365c2644ddc553bb9ed863ebefa"}, + "certifi": {:hex, :certifi, "2.17.0", "835748414307e15e05b17d0e518190228ce648b08d569a5cc93a85a40f3e5c9b", [:rebar3], [], "hexpm", "8122798a17f0293c80daada25d0f81c7f4d708c73fef782c7c9b1950e26e4d21"}, + "cowboy": {:hex, :cowboy, "2.19.0", "78b9d92d25a23e56d7341040b67946c04067cd77de966e2372865f673a8e6f59", [:make, :rebar3], [{:cowlib, ">= 2.20.0 and < 3.0.0", [hex: :cowlib, repo: "hexpm", optional: false]}, {:ranch, ">= 1.8.0 and < 3.0.0", [hex: :ranch, repo: "hexpm", optional: false]}], "hexpm", "986dae81f99fcb78ef2d8efc21d738cee189410840eeaf32eca84ada81dcf6d4"}, + "cowboy_telemetry": {:hex, :cowboy_telemetry, "0.4.0", "f239f68b588efa7707abce16a84d0d2acf3a0f50571f8bb7f56a15865aae820c", [:rebar3], [{:cowboy, "~> 2.7", [hex: :cowboy, repo: "hexpm", optional: false]}, {:telemetry, "~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "7d98bac1ee4565d31b62d59f8823dfd8356a169e7fcbb83831b8a5397404c9de"}, + "cowlib": {:hex, :cowlib, "2.20.0", "bb525377ba634cd6d68bac7bff5d98571f738806139d335daca827717c5dc172", [:make, :rebar3], [], "hexpm", "7d41a0dd2c093041ff3779ac5fe8a1585a68ec7cb2dd1de0536bdd2452fd7ba1"}, + "credo": {:hex, :credo, "1.7.19", "cc52129665fc7c15143d47838fda0f9cd6dac9ceced7bf4da6f85fcbfe64b12a", [:mix], [{:bunt, "~> 0.2.1 or ~> 1.0", [hex: :bunt, repo: "hexpm", optional: false]}, {:file_system, "~> 0.2 or ~> 1.0", [hex: :file_system, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}], "hexpm", "2d8bc95d5a7bb99dd2613621d4f08c6a3575c3fd4b62e6a2b48a100352a557b8"}, + "decimal": {:hex, :decimal, "2.4.1", "6c0fbede12fb122ba685e9ab41c6a40c129e322b3aa192f9e072e61f3a6ffaf2", [:mix], [], "hexpm", "7e618897933a8455f19a727d7c5e50a2c071a544b700e5e724298ecb4340187f"}, + "earmark_parser": {:hex, :earmark_parser, "1.4.46", "67607a0532e810c6f630a515c548d0b24949643f168cc556303bee4cf96105c7", [:mix], [], "hexpm", "9c44636e8a1c68c62f526b2dcd85d941dbbcee7ab82cf64ba06ce28bef8e89f5"}, "elixir_uuid": {:hex, :elixir_uuid, "1.2.1", "dce506597acb7e6b0daeaff52ff6a9043f5919a4c3315abb4143f0b00378c097", [:mix], [], "hexpm", "f7eba2ea6c3555cea09706492716b0d87397b88946e6380898c2889d68585752"}, - "ex_doc": {:hex, :ex_doc, "0.34.2", "13eedf3844ccdce25cfd837b99bea9ad92c4e511233199440488d217c92571e8", [:mix], [{:earmark_parser, "~> 1.4.39", [hex: :earmark_parser, repo: "hexpm", optional: false]}, {:makeup_c, ">= 0.1.0", [hex: :makeup_c, repo: "hexpm", optional: true]}, {:makeup_elixir, "~> 0.14 or ~> 1.0", [hex: :makeup_elixir, repo: "hexpm", optional: false]}, {:makeup_erlang, "~> 0.1 or ~> 1.0", [hex: :makeup_erlang, repo: "hexpm", optional: false]}, {:makeup_html, ">= 0.1.0", [hex: :makeup_html, repo: "hexpm", optional: true]}], "hexpm", "5ce5f16b41208a50106afed3de6a2ed34f4acfd65715b82a0b84b49d995f95c1"}, - "ex_json_schema": {:hex, :ex_json_schema, "0.9.2", "c9a42e04e70cd70eb11a8903a22e8ec344df16edef4cb8e6ec84ed0caffc9f0f", [:mix], [{:decimal, "~> 2.0", [hex: :decimal, repo: "hexpm", optional: false]}], "hexpm", "4854329cb352b6c01c4c4b8dbfb3be14dc5bea19ea13e0eafade4ff22ba55224"}, - "excoveralls": {:hex, :excoveralls, "0.14.6", "610e921e25b180a8538229ef547957f7e04bd3d3e9a55c7c5b7d24354abbba70", [:mix], [{:hackney, "~> 1.16", [hex: :hackney, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}], "hexpm", "0eceddaa9785cfcefbf3cd37812705f9d8ad34a758e513bb975b081dce4eb11e"}, - "file_system": {:hex, :file_system, "1.0.1", "79e8ceaddb0416f8b8cd02a0127bdbababe7bf4a23d2a395b983c1f8b3f73edd", [:mix], [], "hexpm", "4414d1f38863ddf9120720cd976fce5bdde8e91d8283353f0e31850fa89feb9e"}, - "floki": {:hex, :floki, "0.33.1", "f20f1eb471e726342b45ccb68edb9486729e7df94da403936ea94a794f072781", [:mix], [{:html_entities, "~> 0.5.0", [hex: :html_entities, repo: "hexpm", optional: false]}], "hexpm", "461035fd125f13fdf30f243c85a0b1e50afbec876cbf1ceefe6fddd2e6d712c6"}, - "gollum": {:hex, :new_gollum, "0.5.0", "871dd0ee15c65b38932da5b6eac1413c2be96545d6cf5d6419081ce85a9a883a", [:mix], [{:httpoison, "~> 2.2", [hex: :httpoison, repo: "hexpm", optional: false]}], "hexpm", "81722a31ef162270432fbfc3dbf1f57d08530a9e572a57bc528748942d020f84"}, - "hackney": {:hex, :hackney, "1.20.1", "8d97aec62ddddd757d128bfd1df6c5861093419f8f7a4223823537bad5d064e2", [:rebar3], [{:certifi, "~> 2.12.0", [hex: :certifi, repo: "hexpm", optional: false]}, {:idna, "~> 6.1.0", [hex: :idna, repo: "hexpm", optional: false]}, {:metrics, "~> 1.0.0", [hex: :metrics, repo: "hexpm", optional: false]}, {:mimerl, "~> 1.1", [hex: :mimerl, repo: "hexpm", optional: false]}, {:parse_trans, "3.4.1", [hex: :parse_trans, repo: "hexpm", optional: false]}, {:ssl_verify_fun, "~> 1.1.0", [hex: :ssl_verify_fun, repo: "hexpm", optional: false]}, {:unicode_util_compat, "~> 0.7.0", [hex: :unicode_util_compat, repo: "hexpm", optional: false]}], "hexpm", "fe9094e5f1a2a2c0a7d10918fee36bfec0ec2a979994cff8cfe8058cd9af38e3"}, - "html_entities": {:hex, :html_entities, "0.5.2", "9e47e70598da7de2a9ff6af8758399251db6dbb7eebe2b013f2bbd2515895c3c", [:mix], [], "hexpm", "c53ba390403485615623b9531e97696f076ed415e8d8058b1dbaa28181f4fdcc"}, - "httpoison": {:hex, :httpoison, "2.2.1", "87b7ed6d95db0389f7df02779644171d7319d319178f6680438167d7b69b1f3d", [:mix], [{:hackney, "~> 1.17", [hex: :hackney, repo: "hexpm", optional: false]}], "hexpm", "51364e6d2f429d80e14fe4b5f8e39719cacd03eb3f9a9286e61e216feac2d2df"}, - "idna": {:hex, :idna, "6.1.1", "8a63070e9f7d0c62eb9d9fcb360a7de382448200fbbd1b106cc96d3d8099df8d", [:rebar3], [{:unicode_util_compat, "~>0.7.0", [hex: :unicode_util_compat, repo: "hexpm", optional: false]}], "hexpm", "92376eb7894412ed19ac475e4a86f7b413c1b9fbb5bd16dccd57934157944cea"}, - "jason": {:hex, :jason, "1.4.4", "b9226785a9aa77b6857ca22832cffa5d5011a667207eb2a0ad56adb5db443b8a", [:mix], [{:decimal, "~> 1.0 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: true]}], "hexpm", "c5eb0cab91f094599f94d55bc63409236a8ec69a21a67814529e8d5f6cc90b3b"}, - "logger_file_backend": {:hex, :logger_file_backend, "0.0.12", "5afaa76a0cb6123cd19900c0f414044cfc46c24c6a1b80842a9b0e7f6c755e57", [:mix], [], "hexpm", "7335cc4e186a3804f9d3651f2fb42243a11748f1e384421bdd17623ed53fed79"}, - "makeup": {:hex, :makeup, "1.1.2", "9ba8837913bdf757787e71c1581c21f9d2455f4dd04cfca785c70bbfff1a76a3", [:mix], [{:nimble_parsec, "~> 1.2.2 or ~> 1.3", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm", "cce1566b81fbcbd21eca8ffe808f33b221f9eee2cbc7a1706fc3da9ff18e6cac"}, - "makeup_elixir": {:hex, :makeup_elixir, "0.16.2", "627e84b8e8bf22e60a2579dad15067c755531fea049ae26ef1020cad58fe9578", [:mix], [{:makeup, "~> 1.0", [hex: :makeup, repo: "hexpm", optional: false]}, {:nimble_parsec, "~> 1.2.3 or ~> 1.3", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm", "41193978704763f6bbe6cc2758b84909e62984c7752b3784bd3c218bb341706b"}, - "makeup_erlang": {:hex, :makeup_erlang, "1.0.1", "c7f58c120b2b5aa5fd80d540a89fdf866ed42f1f3994e4fe189abebeab610839", [:mix], [{:makeup, "~> 1.0", [hex: :makeup, repo: "hexpm", optional: false]}], "hexpm", "8a89a1eeccc2d798d6ea15496a6e4870b75e014d1af514b1b71fa33134f57814"}, - "meck": {:hex, :meck, "0.9.2", "85ccbab053f1db86c7ca240e9fc718170ee5bda03810a6292b5306bf31bae5f5", [:rebar3], [], "hexpm", "81344f561357dc40a8344afa53767c32669153355b626ea9fcbc8da6b3045826"}, - "metrics": {:hex, :metrics, "1.0.1", "25f094dea2cda98213cecc3aeff09e940299d950904393b2a29d191c346a8486", [:rebar3], [], "hexpm", "69b09adddc4f74a40716ae54d140f93beb0fb8978d8636eaded0c31b6f099f16"}, - "mime": {:hex, :mime, "1.6.0", "dabde576a497cef4bbdd60aceee8160e02a6c89250d6c0b29e56c0dfb00db3d2", [:mix], [], "hexpm", "31a1a8613f8321143dde1dafc36006a17d28d02bdfecb9e95a880fa7aabd19a7"}, - "mimerl": {:hex, :mimerl, "1.2.0", "67e2d3f571088d5cfd3e550c383094b47159f3eee8ffa08e64106cdf5e981be3", [:rebar3], [], "hexpm", "f278585650aa581986264638ebf698f8bb19df297f66ad91b18910dfc6e19323"}, - "nimble_parsec": {:hex, :nimble_parsec, "1.4.0", "51f9b613ea62cfa97b25ccc2c1b4216e81df970acd8e16e8d1bdc58fef21370d", [:mix], [], "hexpm", "9c565862810fb383e9838c1dd2d7d2c437b3d13b267414ba6af33e50d2d1cf28"}, - "parse_trans": {:hex, :parse_trans, "3.4.1", "6e6aa8167cb44cc8f39441d05193be6e6f4e7c2946cb2759f015f8c56b76e5ff", [:rebar3], [], "hexpm", "620a406ce75dada827b82e453c19cf06776be266f5a67cff34e1ef2cbb60e49a"}, - "phoenix": {:hex, :phoenix, "1.6.16", "e5bdd18c7a06da5852a25c7befb72246de4ddc289182285f8685a40b7b5f5451", [:mix], [{:castore, ">= 0.0.0", [hex: :castore, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:phoenix_pubsub, "~> 2.0", [hex: :phoenix_pubsub, repo: "hexpm", optional: false]}, {:phoenix_view, "~> 1.0 or ~> 2.0", [hex: :phoenix_view, repo: "hexpm", optional: false]}, {:plug, "~> 1.10", [hex: :plug, repo: "hexpm", optional: false]}, {:plug_cowboy, "~> 2.2", [hex: :plug_cowboy, repo: "hexpm", optional: true]}, {:plug_crypto, "~> 1.2", [hex: :plug_crypto, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "e15989ff34f670a96b95ef6d1d25bad0d9c50df5df40b671d8f4a669e050ac39"}, - "phoenix_pubsub": {:hex, :phoenix_pubsub, "2.1.1", "ba04e489ef03763bf28a17eb2eaddc2c20c6d217e2150a61e3298b0f4c2012b5", [:mix], [], "hexpm", "81367c6d1eea5878ad726be80808eb5a787a23dee699f96e72b1109c57cdd8d9"}, - "phoenix_template": {:hex, :phoenix_template, "1.0.1", "85f79e3ad1b0180abb43f9725973e3b8c2c3354a87245f91431eec60553ed3ef", [:mix], [{:phoenix_html, "~> 2.14.2 or ~> 3.0", [hex: :phoenix_html, repo: "hexpm", optional: true]}], "hexpm", "157dc078f6226334c91cb32c1865bf3911686f8bcd6bcff86736f6253e6993ee"}, - "phoenix_view": {:hex, :phoenix_view, "2.0.2", "6bd4d2fd595ef80d33b439ede6a19326b78f0f1d8d62b9a318e3d9c1af351098", [:mix], [{:phoenix_html, "~> 2.14.2 or ~> 3.0", [hex: :phoenix_html, repo: "hexpm", optional: true]}, {:phoenix_template, "~> 1.0", [hex: :phoenix_template, repo: "hexpm", optional: false]}], "hexpm", "a929e7230ea5c7ee0e149ffcf44ce7cf7f4b6d2bfe1752dd7c084cdff152d36f"}, - "plug": {:hex, :plug, "1.12.1", "645678c800601d8d9f27ad1aebba1fdb9ce5b2623ddb961a074da0b96c35187d", [:mix], [{:mime, "~> 1.0 or ~> 2.0", [hex: :mime, repo: "hexpm", optional: false]}, {:plug_crypto, "~> 1.1.1 or ~> 1.2", [hex: :plug_crypto, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4.3 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "d57e799a777bc20494b784966dc5fbda91eb4a09f571f76545b72a634ce0d30b"}, - "plug_cowboy": {:hex, :plug_cowboy, "2.5.1", "7cc96ff645158a94cf3ec9744464414f02287f832d6847079adfe0b58761cbd0", [:mix], [{:cowboy, "~> 2.7", [hex: :cowboy, repo: "hexpm", optional: false]}, {:cowboy_telemetry, "~> 0.3", [hex: :cowboy_telemetry, repo: "hexpm", optional: false]}, {:plug, "~> 1.7", [hex: :plug, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "107d0a5865fa92bcb48e631cc0729ae9ccfa0a9f9a1bd8f01acb513abf1c2d64"}, - "plug_crypto": {:hex, :plug_crypto, "1.2.2", "05654514ac717ff3a1843204b424477d9e60c143406aa94daf2274fdd280794d", [:mix], [], "hexpm", "87631c7ad914a5a445f0a3809f99b079113ae4ed4b867348dd9eec288cecb6db"}, - "poison": {:hex, :poison, "3.1.0", "d9eb636610e096f86f25d9a46f35a9facac35609a7591b3be3326e99a0484665", [:mix], [], "hexpm", "fec8660eb7733ee4117b85f55799fd3833eb769a6df71ccf8903e8dc5447cfce"}, - "ranch": {:hex, :ranch, "1.8.0", "8c7a100a139fd57f17327b6413e4167ac559fbc04ca7448e9be9057311597a1d", [:make, :rebar3], [], "hexpm", "49fbcfd3682fab1f5d109351b61257676da1a2fdbe295904176d5e521a2ddfe5"}, - "redirect": {:hex, :redirect, "0.4.0", "98b46053504ee517bc3ad2fd04c064b64b48d339e1e18266355b30c4f8bb52b0", [:mix], [{:phoenix, "~> 1.4", [hex: :phoenix, repo: "hexpm", optional: false]}, {:plug, "~> 1.8.3 or ~> 1.9", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm", "dfa29a8ecbad066ed0b73b34611cf24c78101719737f37bdf750f39197d67b97"}, + "ex_doc": {:hex, :ex_doc, "0.40.4", "66f2e42bf588594d5a8aab31cad87f2ddad09d0da1b1a2f379340ec2c2e497cb", [:mix], [{:earmark_parser, "~> 1.4.46", [hex: :earmark_parser, repo: "hexpm", optional: false]}, {:makeup_c, ">= 0.1.0", [hex: :makeup_c, repo: "hexpm", optional: true]}, {:makeup_elixir, "~> 0.14 or ~> 1.0", [hex: :makeup_elixir, repo: "hexpm", optional: false]}, {:makeup_erlang, "~> 0.1 or ~> 1.0", [hex: :makeup_erlang, repo: "hexpm", optional: false]}, {:makeup_html, ">= 0.1.0", [hex: :makeup_html, repo: "hexpm", optional: true]}], "hexpm", "6222b9e423d76584ee34df2c82a5ed72c2d53dc153f7f483ad28b378694186cc"}, + "ex_json_schema": {:hex, :ex_json_schema, "0.11.3", "0f9128fde2a46976c21d1ed9506bf8462776ac9e333e7b42ececcdbf7433a1da", [:mix], [{:decimal, "~> 2.0", [hex: :decimal, repo: "hexpm", optional: false]}], "hexpm", "0e8e16866bc14339b1bf1e441f1d9231031c8fe2cf45fa5a988a8b685de8c002"}, + "excoveralls": {:hex, :excoveralls, "0.18.5", "e229d0a65982613332ec30f07940038fe451a2e5b29bce2a5022165f0c9b157e", [:mix], [{:castore, "~> 1.0", [hex: :castore, repo: "hexpm", optional: true]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}], "hexpm", "523fe8a15603f86d64852aab2abe8ddbd78e68579c8525ae765facc5eae01562"}, + "file_system": {:hex, :file_system, "1.1.1", "31864f4685b0148f25bd3fbef2b1228457c0c89024ad67f7a81a3ffbc0bbad3a", [:mix], [], "hexpm", "7a15ff97dfe526aeefb090a7a9d3d03aa907e100e262a0f8f7746b78f8f87a5d"}, + "floki": {:hex, :floki, "0.38.4", "10f98971e892aed2c2f1b3a0f928e488e3797e1c6dd3dfd98db40b14e9a78bcf", [:mix], [], "hexpm", "bdb34645eee8e79845c7edaca2d4099a52804ee4d4a3ecc683a69451f0244973"}, + "gollum": {:git, "https://github.com/cmeon/gollum.git", "ed5b76594424f5705565459857519a44da4d7a6f", [branch: "update-dependencies"]}, + "h2": {:hex, :h2, "0.12.0", "f393539ee2728f8118fb2024b6d5f3e2c45e40ceb31b18b4e9bf5e50d028f80f", [:rebar3], [], "hexpm", "beaafc93c54cdc5d623247334d3970cdf4bc66b6b8b296b74ba1d7c7513c3dfc"}, + "hackney": {:hex, :hackney, "4.7.4", "8fe2ddaa3ca27de99d68e682d72b66d07d2331da680f77c8000580a0122c69e6", [:rebar3], [{:certifi, "~> 2.17.0", [hex: :certifi, repo: "hexpm", optional: false]}, {:h2, "~> 0.12.0", [hex: :h2, repo: "hexpm", optional: false]}, {:idna, "~> 7.1.0", [hex: :idna, repo: "hexpm", optional: false]}, {:mimerl, "~> 1.5", [hex: :mimerl, repo: "hexpm", optional: false]}, {:parse_trans, "3.4.2", [hex: :parse_trans, repo: "hexpm", optional: false]}, {:ssl_verify_fun, "~> 1.1.0", [hex: :ssl_verify_fun, repo: "hexpm", optional: false]}, {:webtransport, "~> 0.4.5", [hex: :webtransport, repo: "hexpm", optional: false]}], "hexpm", "d07d7e1358353ab6cc75132f058c155287f3e013d43f709fbb79d79eeab98195"}, + "httpoison": {:hex, :httpoison, "3.0.0", "8566a933bb9175236d1ec335978445b67cd1f5b5d3ead6ca4b80be469d41f5d9", [:mix], [{:hackney, "~> 4.0", [hex: :hackney, repo: "hexpm", optional: false]}], "hexpm", "9130197b7658901c493d6fcfb842fb9676300fa8a6c8ed058c8889cf1a77f3c2"}, + "idna": {:hex, :idna, "7.1.0", "1067a13043538129602d2f2ce6899d8713125c7d19734aa557ce2e3ea55bd4f1", [:rebar3], [], "hexpm", "6ae959a025bf36df61a8cab8508d9654891b5426a84c44d82deaffd6ddf8c71f"}, + "jason": {:hex, :jason, "1.4.5", "2e3a008590b0b8d7388c20293e9dcc9cf3e5d642fd2a114e4cbbb52e595d940a", [:mix], [{:decimal, "~> 1.0 or ~> 2.0 or ~> 3.0", [hex: :decimal, repo: "hexpm", optional: true]}], "hexpm", "b0c823996102bcd0239b3c2444eb00409b72f6a140c1950bc8b457d836b30684"}, + "logger_file_backend": {:hex, :logger_file_backend, "0.0.14", "774bb661f1c3fed51b624d2859180c01e386eb1273dc22de4f4a155ef749a602", [:mix], [], "hexpm", "071354a18196468f3904ef09413af20971d55164267427f6257b52cfba03f9e6"}, + "makeup": {:hex, :makeup, "1.2.2", "882d46dc0905e9ff7abf2aab61a7e6b3dcc555533977d8a23b06019e6c89ac94", [:mix], [{:nimble_parsec, "~> 1.4", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm", "9a1a24e5b343b8ae16abea0822c10a6f75da27af7fa802ada5251f7579bfccfa"}, + "makeup_elixir": {:hex, :makeup_elixir, "1.0.1", "e928a4f984e795e41e3abd27bfc09f51db16ab8ba1aebdba2b3a575437efafc2", [:mix], [{:makeup, "~> 1.0", [hex: :makeup, repo: "hexpm", optional: false]}, {:nimble_parsec, "~> 1.2.3 or ~> 1.3", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm", "7284900d412a3e5cfd97fdaed4f5ed389b8f2b4cb49efc0eb3bd10e2febf9507"}, + "makeup_erlang": {:hex, :makeup_erlang, "1.1.0", "835f7e60792e08824cda445639555d7bf1bbbddb1b60b306e33cb6f6db24dc74", [:mix], [{:makeup, "~> 1.0", [hex: :makeup, repo: "hexpm", optional: false]}], "hexpm", "1cd6780fb1dd1a03979abaed0fe82712b0625118fd5257d3ebbf73f960c73c3c"}, + "mdex": {:hex, :mdex, "0.13.5", "c1c94d230ccaab01ad0c68090d3b31613c10ece1844f32b55895da4ce0c63029", [:mix], [{:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}, {:lumis, "~> 0.1", [hex: :lumis, repo: "hexpm", optional: true]}, {:mdex_native, ">= 0.2.6", [hex: :mdex_native, repo: "hexpm", optional: false]}, {:nimble_options, "~> 1.0", [hex: :nimble_options, repo: "hexpm", optional: false]}, {:nimble_parsec, "~> 1.0", [hex: :nimble_parsec, repo: "hexpm", optional: false]}, {:phoenix_live_view, "~> 0.20.0 or ~> 1.0", [hex: :phoenix_live_view, repo: "hexpm", optional: true]}], "hexpm", "c57409fb6b34fbc58fbce0a6da670c9a4b5a2e94f86abdc56e9e213ed74620f2"}, + "mdex_native": {:hex, :mdex_native, "0.2.8", "20b7cbf330c1ca81b8da4132b8d01952cded11f6dfc2abe8fef25c13681b15e4", [:mix], [{:rustler, "~> 0.32", [hex: :rustler, repo: "hexpm", optional: true]}, {:rustler_precompiled, "~> 0.8", [hex: :rustler_precompiled, repo: "hexpm", optional: false]}], "hexpm", "004a5565b6c96a06400901eb1e4e603585e00b23262d3f595c3f4aa38b83ef66"}, + "meck": {:hex, :meck, "1.2.0", "c3618447506589a2931be80d006ea883aff156dd56e8822f7c1bb49896e94edb", [:rebar3], [], "hexpm", "a2cfd08306ef4992db7cd2aa3521b797456abec9f562f3f0341633052dee4af9"}, + "mime": {:hex, :mime, "2.0.7", "b8d739037be7cd402aee1ba0306edfdef982687ee7e9859bee6198c1e7e2f128", [:mix], [], "hexpm", "6171188e399ee16023ffc5b76ce445eb6d9672e2e241d2df6050f3c771e80ccd"}, + "mimerl": {:hex, :mimerl, "1.5.0", "f35aca6f23242339b3666e0ac0702379e362b469d0aea167f6cc713547e777ed", [:rebar3], [], "hexpm", "db648ce065bae14ea84ca8b5dd123f42f49417cef693541110bf6f9e9be9ecc4"}, + "nimble_options": {:hex, :nimble_options, "1.1.1", "e3a492d54d85fc3fd7c5baf411d9d2852922f66e69476317787a7b2bb000a61b", [:mix], [], "hexpm", "821b2470ca9442c4b6984882fe9bb0389371b8ddec4d45a9504f00a66f650b44"}, + "nimble_parsec": {:hex, :nimble_parsec, "1.4.2", "8efba0122db06df95bfaa78f791344a89352ba04baedd3849593bfce4d0dc1c6", [:mix], [], "hexpm", "4b21398942dda052b403bbe1da991ccd03a053668d147d53fb8c4e0efe09c973"}, + "parse_trans": {:hex, :parse_trans, "3.4.2", "c352ddc1a0d5e54f9b1654d45f9c432eef76f9cea371c55ddff769ef688fdb74", [:rebar3], [], "hexpm", "4c25347de3b7c35732d32e69ab43d1ceee0beae3f3b3ade1b59cbd3dd224d9ca"}, + "plug": {:hex, :plug, "1.20.3", "56c480c633ec2ce10140e236e15233bf576e1d323887d7c96711bd02ab5160db", [:mix], [{:mime, "~> 1.0 or ~> 2.0", [hex: :mime, repo: "hexpm", optional: false]}, {:plug_crypto, "~> 1.1.1 or ~> 1.2 or ~> 2.0", [hex: :plug_crypto, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4.3 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "be266aee1b8536ef6409d58cf39a3121319f0ec47cfa1b24024485aa0e76ad76"}, + "plug_cowboy": {:hex, :plug_cowboy, "2.9.0", "87e21e0d9054ced99c36d128f49e3ea2cd8b745fffb97de50bff99706087af4f", [:mix], [{:cowboy, "~> 2.7", [hex: :cowboy, repo: "hexpm", optional: false]}, {:cowboy_telemetry, "~> 0.3", [hex: :cowboy_telemetry, repo: "hexpm", optional: false]}, {:plug, "~> 1.18", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm", "2002bafba4f3a45b55a58e68d70211b153a7ed18d37edb1ceb6e96e7a92c422e"}, + "plug_crypto": {:hex, :plug_crypto, "2.2.0", "144014737daaf485407f5ed77daeaad74d651b216a28c87543f8cc7043f8efc8", [:mix], [], "hexpm", "83a95744ab1c75876542b6fab135fcc176280e0f301a111c1f757fddcec95d2c"}, + "poison": {:hex, :poison, "6.0.0", "9bbe86722355e36ffb62c51a552719534257ba53f3271dacd20fbbd6621a583a", [:mix], [{:decimal, "~> 2.1", [hex: :decimal, repo: "hexpm", optional: true]}], "hexpm", "bb9064632b94775a3964642d6a78281c07b7be1319e0016e1643790704e739a2"}, + "quic": {:hex, :quic, "1.8.2", "c315176d2c4fad0725e52f2a8033b96d01c4fecfc7e6a7333615ff10041142a5", [:rebar3], [], "hexpm", "274d2f41ee9c00d8d6415248df9fb5637381fe69b5387771dc37e91038a65479"}, + "ranch": {:hex, :ranch, "2.3.0", "7de7b041a9a6a5091a3aa5898d66c0564be671d87db4f9d63b1b5ee775b097df", [:make, :rebar3], [], "hexpm", "6168ec49409d982f7cfbd83dd083144f6cbe67caa4036551d2f0a3ad67c9d023"}, + "rustler_precompiled": {:hex, :rustler_precompiled, "0.9.0", "3a052eda09f3d2436364645cc1f13279cf95db310eb0c17b0d8f25484b233aa0", [:mix], [{:rustler, "~> 0.23", [hex: :rustler, repo: "hexpm", optional: true]}], "hexpm", "471d97315bd3bf7b64623418b3693eedd8e47de3d1cb79a0ac8f9da7d770d94c"}, "ssl_verify_fun": {:hex, :ssl_verify_fun, "1.1.7", "354c321cf377240c7b8716899e182ce4890c5938111a1296add3ec74cf1715df", [:make, :mix, :rebar3], [], "hexpm", "fe4c190e8f37401d30167c8c405eda19469f34577987c76dde613e838bbc67f8"}, - "telemetry": {:hex, :telemetry, "0.4.3", "a06428a514bdbc63293cd9a6263aad00ddeb66f608163bdec7c8995784080818", [:rebar3], [], "hexpm", "eb72b8365ffda5bed68a620d1da88525e326cb82a75ee61354fc24b844768041"}, - "unicode_util_compat": {:hex, :unicode_util_compat, "0.7.0", "bc84380c9ab48177092f43ac89e4dfa2c6d62b40b8bd132b1059ecc7232f9a78", [:rebar3], [], "hexpm", "25eee6d67df61960cf6a794239566599b09e17e668d3700247bc498638152521"}, + "telemetry": {:hex, :telemetry, "1.4.2", "a0cb522801dffb1c49fe6e30561badffc7b6d0e180db1300df759faa22062855", [:rebar3], [], "hexpm", "928f6495066506077862c0d1646609eed891a4326bee3126ba54b60af61febb1"}, + "webtransport": {:hex, :webtransport, "0.4.5", "0e387202bbe707389fe81373ef8c56faa9d5aa321bb4800fa4765ee7c1399785", [:rebar3], [{:h2, "~> 0.12", [hex: :h2, repo: "hexpm", optional: false]}, {:quic, "~> 1.8.0", [hex: :quic, repo: "hexpm", optional: false]}], "hexpm", "bcb512239e48e551d5bd5c667312a9a7de4f29b89d84b1d33c5af44e1f3f730d"}, "yamerl": {:hex, :yamerl, "0.10.0", "4ff81fee2f1f6a46f1700c0d880b24d193ddb74bd14ef42cb0bcf46e81ef2f8e", [:rebar3], [], "hexpm", "346adb2963f1051dc837a2364e4acf6eb7d80097c0f53cbdc3046ec8ec4b4e6e"}, - "yaml_elixir": {:hex, :yaml_elixir, "2.9.0", "9a256da867b37b8d2c1ffd5d9de373a4fda77a32a45b452f1708508ba7bbcb53", [:mix], [{:yamerl, "~> 0.10", [hex: :yamerl, repo: "hexpm", optional: false]}], "hexpm", "0cb0e7d4c56f5e99a6253ed1a670ed0e39c13fc45a6da054033928607ac08dfc"}, + "yaml_elixir": {:hex, :yaml_elixir, "2.12.2", "9dd1330fb4cd9a36a7b0f502e5b12486eff632792ee4a5f0eba52a4d4ec32c9c", [:mix], [{:yamerl, "~> 0.10", [hex: :yamerl, repo: "hexpm", optional: false]}], "hexpm", "e7c1b10122f973e6558462d51c39026ba0e14afbc6745318e990ea82cfe9e159"}, } diff --git a/test/api_test.exs b/test/api_test.exs index 9c96846a..429fc664 100644 --- a/test/api_test.exs +++ b/test/api_test.exs @@ -1,6 +1,6 @@ defmodule APITest do use ExUnit.Case, async: false - use Plug.Test + import Plug.Test @opts Crawly.API.Router.init([]) diff --git a/test/engine_test.exs b/test/engine_test.exs index d898baee..56fb1dea 100644 --- a/test/engine_test.exs +++ b/test/engine_test.exs @@ -58,65 +58,63 @@ defmodule EngineTest do end test ":log_to_file allows for logging to log file" do - :meck.expect(TestSpider, :override_settings, fn -> - [log_dir: "/my_tmp_dir", log_to_file: true] - end) + tmp_dir = System.tmp_dir!() |> Path.join("crawly_test_logs") + crawl_id = "test_crawl_#{System.unique_integer([:positive])}" - :meck.expect(Logger, :configure_backend, fn {_, :debug}, opts -> - log_file_path = Keyword.get(opts, :path) - assert log_file_path =~ "TestSpider" - assert log_file_path =~ "/my_tmp_dir" + :meck.expect(TestSpider, :override_settings, fn -> + [log_dir: tmp_dir, log_to_file: true] end) Crawly.Engine.refresh_spider_list() + Crawly.Engine.start_spider(TestSpider, crawl_id: crawl_id) - # test a started spider - Crawly.Engine.start_spider(TestSpider) + handler_id = :"crawly_TestSpider_#{crawl_id}" + assert {:ok, config} = :logger.get_handler_config(handler_id) + assert to_string(config.config.file) =~ tmp_dir + assert to_string(config.config.file) =~ "TestSpider" + + Crawly.Engine.stop_spider(TestSpider) - assert :meck.num_calls(Logger, :configure_backend, :_) == 1 + assert {:error, {:not_found, ^handler_id}} = + :logger.get_handler_config(handler_id) end - test "LoggerFileBackend is not configured when module is not loaded" do + test ":logger handler is not added when log_to_file is disabled" do + crawl_id = "no_log_#{System.unique_integer([:positive])}" + :meck.expect(TestSpider, :override_settings, fn -> - [log_dir: "/my_tmp_dir", log_to_file: true] + [log_to_file: false] end) - :meck.expect(Crawly.Utils, :ensure_loaded?, fn _ -> - false - end) + Crawly.Engine.refresh_spider_list() + Crawly.Engine.start_spider(TestSpider, crawl_id: crawl_id) - :meck.expect(Logger, :configure_backend, fn {_, :debug}, opts -> - log_file_path = Keyword.get(opts, :path) - assert log_file_path =~ "TestSpider" - assert log_file_path =~ "/my_tmp_dir" - end) + handler_id = :"crawly_TestSpider_#{crawl_id}" - Crawly.Engine.refresh_spider_list() + assert {:error, {:not_found, ^handler_id}} = + :logger.get_handler_config(handler_id) - # test a started spider - Crawly.Engine.start_spider(TestSpider) - assert :meck.num_calls(Logger, :configure_backend, :_) == 0 + Crawly.Engine.stop_spider(TestSpider) end - test "LoggerFileBackend is configured when module is loaded" do + test ":logger handler is configured and added when log_to_file is enabled" do + tmp_dir = System.tmp_dir!() |> Path.join("crawly_test_logs_enabled") + crawl_id = "enabled_#{System.unique_integer([:positive])}" + :meck.expect(TestSpider, :override_settings, fn -> - [log_dir: "/my_tmp_dir", log_to_file: true] + [log_dir: tmp_dir, log_to_file: true] end) - :meck.expect(Crawly.Utils, :ensure_loaded?, fn _ -> - true - end) + Crawly.Engine.refresh_spider_list() + Crawly.Engine.start_spider(TestSpider, crawl_id: crawl_id) - :meck.expect(Logger, :configure_backend, fn {_, :debug}, opts -> - log_file_path = Keyword.get(opts, :path) - assert log_file_path =~ "TestSpider" - assert log_file_path =~ "/my_tmp_dir" - end) + handler_id = :"crawly_TestSpider_#{crawl_id}" + assert {:ok, config} = :logger.get_handler_config(handler_id) + assert to_string(config.config.file) =~ tmp_dir - Crawly.Engine.refresh_spider_list() + Crawly.Engine.stop_spider(TestSpider) - # test a started spider - Crawly.Engine.start_spider(TestSpider) - assert :meck.num_calls(Logger, :configure_backend, :_) == 1 + assert {:error, {:not_found, ^handler_id}} = + :logger.get_handler_config(handler_id) end end diff --git a/test/fetchers/crawly_render_server_test.exs b/test/fetchers/crawly_render_server_test.exs index 2734ec05..2b3a5639 100644 --- a/test/fetchers/crawly_render_server_test.exs +++ b/test/fetchers/crawly_render_server_test.exs @@ -6,7 +6,8 @@ defmodule Crawly.Fetchers.CrawlyRenderServerTest do test "throws an error when base_url is not set" do request = %{ url: "https://example.com", - headers: %{"User-Agent" => "Custom User Agent"} + headers: %{"User-Agent" => "Custom User Agent"}, + options: [] } client_options = [] diff --git a/test/manager_test.exs b/test/manager_test.exs index 60e066e6..340106dc 100644 --- a/test/manager_test.exs +++ b/test/manager_test.exs @@ -1,6 +1,8 @@ defmodule ManagerTest do use ExUnit.Case, async: false + import TestUtils, only: [assert_eventually: 1] + alias Crawly.Engine setup do @@ -76,13 +78,19 @@ defmodule ManagerTest do ) Process.sleep(700) - assert :persistent_term.get(:spider_stop_reason) == :itemcount_limit + + assert_eventually(fn -> + assert :persistent_term.get(:spider_stop_reason) == :itemcount_limit + end) end test "Closespider timeout is respected" do :ok = Crawly.Engine.start_spider(Manager.TestSpider) Process.sleep(600) - assert :persistent_term.get(:spider_stop_reason) == :itemcount_timeout + + assert_eventually(fn -> + assert :persistent_term.get(:spider_stop_reason) == :itemcount_timeout + end) end test "Can't start already started spider" do @@ -95,7 +103,10 @@ defmodule ManagerTest do test "Spider closed callback is called when spider is stopped" do :ok = Crawly.Engine.start_spider(Manager.TestSpider) :ok = Crawly.Engine.stop_spider(Manager.TestSpider, :manual_stop) - assert :persistent_term.get(:spider_stop_reason) == :manual_stop + + assert_eventually(fn -> + assert :persistent_term.get(:spider_stop_reason) == :manual_stop + end) end test "It's possible to start a spider with start_requests" do diff --git a/test/test_utils.ex b/test/test_utils.ex index b5703e42..4045cb5c 100644 --- a/test/test_utils.ex +++ b/test/test_utils.ex @@ -18,6 +18,23 @@ defmodule TestUtils do result end + + def assert_eventually(fun, timeout \\ 2_000, interval \\ 20) do + deadline = System.monotonic_time(:millisecond) + timeout + do_assert_eventually(fun, deadline, interval) + end + + defp do_assert_eventually(fun, deadline, interval) do + fun.() + rescue + error -> + if System.monotonic_time(:millisecond) >= deadline do + reraise error, __STACKTRACE__ + else + Process.sleep(interval) + do_assert_eventually(fun, deadline, interval) + end + end end defmodule TestSpider do