Update the FetchJsons to allow use a function to transform the data.

lib/decentralised_book_index/sync/api_clients/fetch_jsons.ex

@@ -1,13 +1,10 @@
 defmodule DecentralisedBookIndex.Sync.ApiClients.FetchJsons do
+  alias DecentralisedBookIndex.Sync.ApiClients.FetchJsons
   alias DecentralisedBookIndex.Sync.ApiClients.FetchJson
 
-  def get(url) do
-    case FetchJson.get(url) do
-      {:ok, data} ->
-        []
-      {:error, message} -> {:error, message}
-    end
+  require Logger
 
+  def get(url, mapper \\ &FetchJsons.get_data/1) do
     query_results =
       Stream.resource(
         # Initial state:
@@ -20,9 +17,9 @@ defmodule DecentralisedBookIndex.Sync.ApiClients.FetchJsons do
           next ->
             case FetchJson.get(next) do
               {:ok, page} ->
-                {page["data"], get_in(page, ["links", "next"])}
+                {mapper.(page["data"]), get_in(page, ["links", "next"])}
               {:error, message} ->
-                Logger.error("FetchJsons error: #{inspect(message)}")
+                Logger.error("FetchJsons error for #{url}: #{inspect(message)}")
                 {[], nil}
             end
         end,
@@ -32,4 +29,8 @@ defmodule DecentralisedBookIndex.Sync.ApiClients.FetchJsons do
 
     {:ok, query_results}
   end
+
+  def get_data(data) do
+    data
+  end
 end

test/decentralised_book_index/sync/api_clients/fetch_jsons_test.exs

@@ -17,5 +17,21 @@ defmodule DecentralisedBookIndex.Sync.ApiClients.FetchJsonsTest do
       assert {:ok, records} = FetchJsons.get("#{@test_server_endpoint}/api/v1/json/author")
       assert length(records) == 11
     end
+
+    test "get authors names from API" do
+      for number <- 1..11 do
+        Metadata.create_author!("Author#{number}", "An description#{number}")
+      end
+
+      get_author_names = fn data ->
+        Enum.map(data, fn author -> author["attributes"]["name"] end)
+      end
+
+      assert {:ok, records} =
+        FetchJsons.get("#{@test_server_endpoint}/api/v1/json/author", get_author_names)
+
+      assert length(records) == 11
+      assert Enum.all?(records, fn author -> String.contains?(author, "Author") end)
+    end
   end
 end