Add the json fetchers to scrape the API.

lib/decentralised_book_index/sync/api_clients/fetch_json.ex

@@ -0,0 +1,13 @@
+defmodule DecentralisedBookIndex.Sync.ApiClients.FetchJson do
+  def get(url) do
+    case Req.get(url) do
+      {:ok, data} ->
+        if data.status == 200 do
+          {:ok, data.body}
+        else
+          {:error, data}
+        end
+      {:error, message} -> {:error, message}
+    end
+  end
+end

lib/decentralised_book_index/sync/api_clients/fetch_jsons.ex

@@ -0,0 +1,35 @@
+defmodule DecentralisedBookIndex.Sync.ApiClients.FetchJsons do
+  alias DecentralisedBookIndex.Sync.ApiClients.FetchJson
+
+  def get(url) do
+    case FetchJson.get(url) do
+      {:ok, data} ->
+        []
+      {:error, message} -> {:error, message}
+    end
+
+    query_results =
+      Stream.resource(
+        # Initial state:
+        fn -> url end,
+        fn
+          # Stop the stream if no more data
+          next when is_nil(next) ->
+            {:halt, nil}
+
+          next ->
+            case FetchJson.get(next) do
+              {:ok, page} ->
+                {page["data"], get_in(page, ["links", "next"])}
+              {:error, message} ->
+                Logger.error("FetchJsons error: #{inspect(message)}")
+                {[], nil}
+            end
+        end,
+        fn _ -> :ok end
+      )
+      |> Enum.to_list()
+
+    {:ok, query_results}
+  end
+end

test/decentralised_book_index/sync/api_clients/fetch_json_test.exs

@@ -0,0 +1,18 @@
+defmodule DecentralisedBookIndex.Sync.ApiClients.FetchJsonTest do
+  use DecentralisedBookIndex.DataCase
+
+  alias DecentralisedBookIndex.Sync.ApiClients.FetchJson
+  alias DecentralisedBookIndex.Metadata
+
+  alias DecentralisedBookIndex.TestEndpoints
+  @test_server_endpoint TestEndpoints.test_api_endpoint()
+
+  describe "authors api" do
+    test "get an author" do
+      {:ok, author} = Metadata.create_author("Author", "An description")
+
+      assert {:ok, data} = FetchJson.get("#{@test_server_endpoint}/api/v1/json/author/#{author.id}")
+      assert data["data"]["id"] == author.id
+    end
+  end
+end

test/decentralised_book_index/sync/api_clients/fetch_jsons_test.exs

@@ -0,0 +1,21 @@
+defmodule DecentralisedBookIndex.Sync.ApiClients.FetchJsonsTest do
+  use DecentralisedBookIndex.DataCase
+
+  alias DecentralisedBookIndex.Sync.ApiClients.FetchJsons
+  alias DecentralisedBookIndex.Metadata
+
+  alias DecentralisedBookIndex.TestEndpoints
+
+  @test_server_endpoint TestEndpoints.test_api_endpoint()
+
+  describe "authors api" do
+    test "get authors" do
+      for number <- 1..11 do
+        Metadata.create_author!("Author#{number}", "An description#{number}")
+      end
+
+      assert {:ok, records} = FetchJsons.get("#{@test_server_endpoint}/api/v1/json/author")
+      assert length(records) == 11
+    end
+  end
+end