Update the FetchJsons to allow use a function to transform the data.

lib/decentralised_book_index/sync/api_clients/fetch_jsons.ex

@@ -1,13 +1,10 @@
 defmodule DecentralisedBookIndex.Sync.ApiClients.FetchJsons do
+  alias DecentralisedBookIndex.Sync.ApiClients.FetchJsons
   alias DecentralisedBookIndex.Sync.ApiClients.FetchJson
 
-  def get(url) do
-    case FetchJson.get(url) do
-      {:ok, data} ->
-        []
-      {:error, message} -> {:error, message}
-    end
+  require Logger
 
+  def get(url, mapper \\ &FetchJsons.get_data/1) do
     query_results =
       Stream.resource(
         # Initial state:
@@ -20,9 +17,9 @@ defmodule DecentralisedBookIndex.Sync.ApiClients.FetchJsons do
           next ->
             case FetchJson.get(next) do
               {:ok, page} ->
-                {page["data"], get_in(page, ["links", "next"])}
+                {mapper.(page["data"]), get_in(page, ["links", "next"])}
               {:error, message} ->
-                Logger.error("FetchJsons error: #{inspect(message)}")
+                Logger.error("FetchJsons error for #{url}: #{inspect(message)}")
                 {[], nil}
             end
         end,
@@ -32,4 +29,8 @@ defmodule DecentralisedBookIndex.Sync.ApiClients.FetchJsons do
 
     {:ok, query_results}
   end
+
+  def get_data(data) do
+    data
+  end
 end