From 57ead52ba250544138abe84c3913bd533eab7bef Mon Sep 17 00:00:00 2001
From: KKlochko <KKlochko@protonmail.com>
Date: Tue, 18 Mar 2025 18:34:43 +0200
Subject: [PATCH] Add the json fetchers to scrape the API.

---
 .../sync/api_clients/fetch_json.ex            | 13 +++++++
 .../sync/api_clients/fetch_jsons.ex           | 35 +++++++++++++++++++
 .../sync/api_clients/fetch_json_test.exs      | 18 ++++++++++
 .../sync/api_clients/fetch_jsons_test.exs     | 21 +++++++++++
 4 files changed, 87 insertions(+)
 create mode 100644 lib/decentralised_book_index/sync/api_clients/fetch_json.ex
 create mode 100644 lib/decentralised_book_index/sync/api_clients/fetch_jsons.ex
 create mode 100644 test/decentralised_book_index/sync/api_clients/fetch_json_test.exs
 create mode 100644 test/decentralised_book_index/sync/api_clients/fetch_jsons_test.exs

diff --git a/lib/decentralised_book_index/sync/api_clients/fetch_json.ex b/lib/decentralised_book_index/sync/api_clients/fetch_json.ex
new file mode 100644
index 0000000..6031c30
--- /dev/null
+++ b/lib/decentralised_book_index/sync/api_clients/fetch_json.ex
@@ -0,0 +1,13 @@
+defmodule DecentralisedBookIndex.Sync.ApiClients.FetchJson do
+  def get(url) do
+    case Req.get(url) do
+      {:ok, data} ->
+        if data.status == 200 do
+          {:ok, data.body}
+        else
+          {:error, data}
+        end
+      {:error, message} -> {:error, message}
+    end
+  end
+end
diff --git a/lib/decentralised_book_index/sync/api_clients/fetch_jsons.ex b/lib/decentralised_book_index/sync/api_clients/fetch_jsons.ex
new file mode 100644
index 0000000..f5b84f1
--- /dev/null
+++ b/lib/decentralised_book_index/sync/api_clients/fetch_jsons.ex
@@ -0,0 +1,35 @@
+defmodule DecentralisedBookIndex.Sync.ApiClients.FetchJsons do
+  alias DecentralisedBookIndex.Sync.ApiClients.FetchJson
+
+  def get(url) do
+    case FetchJson.get(url) do
+      {:ok, data} ->
+        []
+      {:error, message} -> {:error, message}
+    end
+
+    query_results =
+      Stream.resource(
+        # Initial state:
+        fn -> url end,
+        fn
+          # Stop the stream if no more data
+          next when is_nil(next) ->
+            {:halt, nil}
+
+          next ->
+            case FetchJson.get(next) do
+              {:ok, page} ->
+                {page["data"], get_in(page, ["links", "next"])}
+              {:error, message} ->
+                Logger.error("FetchJsons error: #{inspect(message)}")
+                {[], nil}
+            end
+        end,
+        fn _ -> :ok end
+      )
+      |> Enum.to_list()
+
+    {:ok, query_results}
+  end
+end
diff --git a/test/decentralised_book_index/sync/api_clients/fetch_json_test.exs b/test/decentralised_book_index/sync/api_clients/fetch_json_test.exs
new file mode 100644
index 0000000..985da0a
--- /dev/null
+++ b/test/decentralised_book_index/sync/api_clients/fetch_json_test.exs
@@ -0,0 +1,18 @@
+defmodule DecentralisedBookIndex.Sync.ApiClients.FetchJsonTest do
+  use DecentralisedBookIndex.DataCase
+
+  alias DecentralisedBookIndex.Sync.ApiClients.FetchJson
+  alias DecentralisedBookIndex.Metadata
+
+  alias DecentralisedBookIndex.TestEndpoints
+  @test_server_endpoint TestEndpoints.test_api_endpoint()
+
+  describe "authors api" do
+    test "get an author" do
+      {:ok, author} = Metadata.create_author("Author", "An description")
+
+      assert {:ok, data} = FetchJson.get("#{@test_server_endpoint}/api/v1/json/author/#{author.id}")
+      assert data["data"]["id"] == author.id
+    end
+  end
+end
diff --git a/test/decentralised_book_index/sync/api_clients/fetch_jsons_test.exs b/test/decentralised_book_index/sync/api_clients/fetch_jsons_test.exs
new file mode 100644
index 0000000..be99d41
--- /dev/null
+++ b/test/decentralised_book_index/sync/api_clients/fetch_jsons_test.exs
@@ -0,0 +1,21 @@
+defmodule DecentralisedBookIndex.Sync.ApiClients.FetchJsonsTest do
+  use DecentralisedBookIndex.DataCase
+
+  alias DecentralisedBookIndex.Sync.ApiClients.FetchJsons
+  alias DecentralisedBookIndex.Metadata
+
+  alias DecentralisedBookIndex.TestEndpoints
+
+  @test_server_endpoint TestEndpoints.test_api_endpoint()
+
+  describe "authors api" do
+    test "get authors" do
+      for number <- 1..11 do
+        Metadata.create_author!("Author#{number}", "An description#{number}")
+      end
+
+      assert {:ok, records} = FetchJsons.get("#{@test_server_endpoint}/api/v1/json/author")
+      assert length(records) == 11
+    end
+  end
+end