diff --git a/lib/decentralised_book_index/metadata/publisher.ex b/lib/decentralised_book_index/metadata/publisher.ex index 38ef791..261fc1d 100644 --- a/lib/decentralised_book_index/metadata/publisher.ex +++ b/lib/decentralised_book_index/metadata/publisher.ex @@ -25,6 +25,10 @@ defmodule DecentralisedBookIndex.Metadata.Publisher do accept [:name] end + create :sync_create do + accept [:id, :name, :inserted_at, :updated_at, :dbi_server_id] + end + read :by_id do argument :id, :uuid, allow_nil?: false get? true @@ -41,17 +45,24 @@ defmodule DecentralisedBookIndex.Metadata.Publisher do pagination offset?: true, default_limit: 10 end + + update :sync do + accept [:name, :inserted_at, :updated_at, :dbi_server_id] + end end attributes do - uuid_primary_key :id + uuid_primary_key :id, writable?: true attribute :name, :string do allow_nil? false public? true end - timestamps() + timestamps() do + writable? true + public? true + end end relationships do diff --git a/lib/decentralised_book_index/sync/data_transformers/publisher_transformer.ex b/lib/decentralised_book_index/sync/data_transformers/publisher_transformer.ex new file mode 100644 index 0000000..77a14e1 --- /dev/null +++ b/lib/decentralised_book_index/sync/data_transformers/publisher_transformer.ex @@ -0,0 +1,22 @@ +defmodule DecentralisedBookIndex.Sync.DataTransformers.PublisherTransformer do + def from_json(json_body) do + attrs = + if Map.has_key?(json_body, "data") do + %{ + id: get_in(json_body, ["data", "id"]), + name: get_in(json_body, ["data", "attributes", "name"]), + inserted_at: get_in(json_body, ["data", "attributes", "inserted_at"]), + updated_at: get_in(json_body, ["data", "attributes", "updated_at"]), + } + else + %{ + id: get_in(json_body, ["id"]), + name: get_in(json_body, ["attributes", "name"]), + inserted_at: get_in(json_body, ["attributes", "inserted_at"]), + updated_at: get_in(json_body, ["attributes", "updated_at"]), + } + end + + {:ok, attrs} + end +end diff --git a/lib/decentralised_book_index/sync/sync/publisher_sync.ex b/lib/decentralised_book_index/sync/sync/publisher_sync.ex new file mode 100644 index 0000000..41d22dc --- /dev/null +++ b/lib/decentralised_book_index/sync/sync/publisher_sync.ex @@ -0,0 +1,29 @@ +defmodule DecentralisedBookIndex.Sync.PublisherSync do + alias DecentralisedBookIndex.Metadata + alias DecentralisedBookIndex.Metadata.Publisher + + def create_update(attrs, server_id) do + case Metadata.get_publisher_by_id(attrs.id) do + {:ok, publisher} -> + attrs = + attrs + |> Map.delete(:id) + + publisher + |> Ash.Changeset.for_update(:sync, attrs) + |> Ash.update!() + + :ok + {:error, %Ash.Error.Query.NotFound{}} -> + attrs = + attrs + |> Map.put(:dbi_server_id, server_id) + + Publisher + |> Ash.Changeset.for_create(:sync_create, attrs) + |> Ash.create!() + + :ok + end + end +end diff --git a/lib/decentralised_book_index/sync/sync_tasks/sync_publishers_task.ex b/lib/decentralised_book_index/sync/sync_tasks/sync_publishers_task.ex new file mode 100644 index 0000000..b63f9e4 --- /dev/null +++ b/lib/decentralised_book_index/sync/sync_tasks/sync_publishers_task.ex @@ -0,0 +1,34 @@ +defmodule DecentralisedBookIndex.SyncTasks.SyncPublishersTask do + alias DecentralisedBookIndex.Sync.ApiClients.FetchJsons + alias DecentralisedBookIndex.Sync.DataTransformers.PublisherTransformer + alias DecentralisedBookIndex.Sync.PublisherSync + + alias DecentralisedBookIndex.Metadata.DBIServer + + def sync(%DBIServer{} = server) do + url = "#{server.url}/api/v1/json/publishers" + FetchJsons.get(url, sync_closure(server)) + + server + end + + def sync_chunk(json_chunk, server_id) do + for json <- json_chunk do + with {:ok, attrs} <- PublisherTransformer.from_json(json), + :ok <- PublisherSync.create_update(attrs, server_id) do + :ok + else + {:error, reason} -> + Logger.error("Pipeline error: #{inspect(reason)}") + end + end + + [] + end + + def sync_closure(server) do + fn json_chunk -> + sync_chunk(json_chunk, server.id) + end + end +end diff --git a/test/decentralised_book_index/sync/data_transformers/publisher_transformer_test.exs b/test/decentralised_book_index/sync/data_transformers/publisher_transformer_test.exs new file mode 100644 index 0000000..4477294 --- /dev/null +++ b/test/decentralised_book_index/sync/data_transformers/publisher_transformer_test.exs @@ -0,0 +1,64 @@ +defmodule DecentralisedBookIndex.Sync.DataTransformers.PublisherTransformerTest do + use ExUnit.Case, async: true + + alias DecentralisedBookIndex.Sync.DataTransformers.PublisherTransformer + alias DecentralisedBookIndex.Metadata.Publisher + + describe "correct transformations" do + test "a json contains correct publisher information" do + json_body = %{ + "data" => %{ + "attributes" => %{ + "inserted_at" => "2025-03-21T09:20:48.791539Z", + "name" => "Publisher", + "updated_at" => "2025-03-21T09:20:48.791539Z" + }, + "id" => "11349865-1b7b-454a-b999-6c4059888a78", + "links" => %{}, + "meta" => %{}, + "relationships" => %{}, + "type" => "publisher" + }, + "jsonapi" => %{"version" => "1.0"}, + "links" => %{ + "self" => + "http://localhost:4000/api/v1/json/publishers/11349865-1b7b-454a-b999-6c4059888a78" + }, + "meta" => %{} + } + + assert {:ok, publisher} = PublisherTransformer.from_json(json_body) + + assert %{ + id: "11349865-1b7b-454a-b999-6c4059888a78", + name: "Publisher", + inserted_at: "2025-03-21T09:20:48.791539Z", + updated_at: "2025-03-21T09:20:48.791539Z" + } = publisher + end + + test "a json doesn't contains publisher information \"data\" attribute" do + json_body = %{ + "attributes" => %{ + "inserted_at" => "2025-03-21T09:20:48.791539Z", + "name" => "Publisher", + "updated_at" => "2025-03-21T09:20:48.791539Z" + }, + "id" => "11349865-1b7b-454a-b999-6c4059888a78", + "links" => %{}, + "meta" => %{}, + "relationships" => %{}, + "type" => "publisher" + } + + assert {:ok, publisher} = PublisherTransformer.from_json(json_body) + + assert %{ + id: "11349865-1b7b-454a-b999-6c4059888a78", + name: "Publisher", + inserted_at: "2025-03-21T09:20:48.791539Z", + updated_at: "2025-03-21T09:20:48.791539Z" + } = publisher + end + end +end diff --git a/test/decentralised_book_index/sync/sync/publisher_sync_test.exs b/test/decentralised_book_index/sync/sync/publisher_sync_test.exs new file mode 100644 index 0000000..ce2832d --- /dev/null +++ b/test/decentralised_book_index/sync/sync/publisher_sync_test.exs @@ -0,0 +1,53 @@ +defmodule DecentralisedBookIndex.Sync.DataTransformers.PublisherSyncTest do + use DecentralisedBookIndex.DataCase, async: true + + alias DecentralisedBookIndex.Sync.PublisherSync + alias DecentralisedBookIndex.Metadata + + alias DecentralisedBookIndex.TestEndpoints + @test_server_endpoint TestEndpoints.test_api_endpoint() + + describe "sync publisher transformations" do + test "a new publisher will be created" do + server = generate(dbi_server(url: @test_server_endpoint)) + + publisher = %{ + id: "11349865-1b7b-454a-b999-6c4059888a78", + name: "Publisher", + inserted_at: "2025-03-21T09:20:48.791539Z", + updated_at: "2025-03-21T09:20:48.791539Z" + } + + {:ok, inserted_at, 0} = DateTime.from_iso8601(publisher[:inserted_at]) + {:ok, updated_at, 0} = DateTime.from_iso8601(publisher[:updated_at]) + + assert :ok = PublisherSync.create_update(publisher, server.id) + assert {:ok, saved_publisher} = Metadata.get_publisher_by_id(publisher.id) + + publisher = + publisher + |> Map.replace(:inserted_at, inserted_at) + |> Map.replace(:updated_at, updated_at) + + assert publisher = saved_publisher + end + + test "update an existing publisher" do + server = generate(dbi_server(url: @test_server_endpoint)) + + {:ok, publisher} = Metadata.create_publisher("Publisher") + + publisher_attrs = %{ + id: publisher.id, + name: "Publisher", + inserted_at: "2025-03-21T09:20:48.791539Z", + updated_at: "2025-03-21T09:20:48.791539Z" + } + + assert :ok = PublisherSync.create_update(publisher_attrs, server.id) + assert {:ok, saved_publisher} = Metadata.get_publisher_by_id(publisher.id) + + assert publisher_attrs = saved_publisher + end + end +end diff --git a/test/decentralised_book_index/sync/sync_tasks/sync_publishers_task_test.exs b/test/decentralised_book_index/sync/sync_tasks/sync_publishers_task_test.exs new file mode 100644 index 0000000..89f12a7 --- /dev/null +++ b/test/decentralised_book_index/sync/sync_tasks/sync_publishers_task_test.exs @@ -0,0 +1,20 @@ +defmodule DecentralisedBookIndex.SyncTasks.SyncPublishersTaskTest do + use DecentralisedBookIndex.DataCase + + alias DecentralisedBookIndex.SyncTasks.SyncPublishersTask + alias DecentralisedBookIndex.Metadata + + alias DecentralisedBookIndex.TestEndpoints + @test_server_endpoint TestEndpoints.test_api_endpoint() + + describe "sync publishers tasks" do + test "sync publisher" do + server = generate(dbi_server(url: @test_server_endpoint)) + + {:ok, _publisher} = Metadata.create_publisher("Publisher") + {:ok, _publisher} = Metadata.create_publisher("Publisher2") + + assert server = SyncPublishersTask.sync(server) + end + end +end