Add the json fetchers to scrape the API.

lib/decentralised_book_index/sync/api_clients/fetch_json.ex

@@ -0,0 +1,13 @@
+defmodule DecentralisedBookIndex.Sync.ApiClients.FetchJson do
+  def get(url) do
+    case Req.get(url) do
+      {:ok, data} ->
+        if data.status == 200 do
+          {:ok, data.body}
+        else
+          {:error, data}
+        end
+      {:error, message} -> {:error, message}
+    end
+  end
+end

lib/decentralised_book_index/sync/api_clients/fetch_jsons.ex

@@ -0,0 +1,35 @@
+defmodule DecentralisedBookIndex.Sync.ApiClients.FetchJsons do
+  alias DecentralisedBookIndex.Sync.ApiClients.FetchJson
+
+  def get(url) do
+    case FetchJson.get(url) do
+      {:ok, data} ->
+        []
+      {:error, message} -> {:error, message}
+    end
+
+    query_results =
+      Stream.resource(
+        # Initial state:
+        fn -> url end,
+        fn
+          # Stop the stream if no more data
+          next when is_nil(next) ->
+            {:halt, nil}
+
+          next ->
+            case FetchJson.get(next) do
+              {:ok, page} ->
+                {page["data"], get_in(page, ["links", "next"])}
+              {:error, message} ->
+                Logger.error("FetchJsons error: #{inspect(message)}")
+                {[], nil}
+            end
+        end,
+        fn _ -> :ok end
+      )
+      |> Enum.to_list()
+
+    {:ok, query_results}
+  end
+end