From 981e1f0b9223ad35a13c83c992b36b0caa7cc08c Mon Sep 17 00:00:00 2001 From: KKlochko Date: Mon, 5 May 2025 21:30:27 +0300 Subject: [PATCH] Add the sync for DBIServer. --- lib/decentralised_book_index/metadata.ex | 2 +- .../metadata/dbi_server.ex | 10 ++- .../dbi_server_transformer.ex | 21 ++++++ .../sync/sync/dbi_server_sync.ex | 31 +++++++++ .../sync/sync_tasks/sync_dbi_server_task.ex | 36 ++++++++++ .../sync/sync_tasks/sync_server_task.ex | 2 + .../dbi_server_transformer_test.exs | 67 +++++++++++++++++++ .../sync/sync/dbi_server_sync_test.exs | 62 +++++++++++++++++ .../sync_tasks/sync_dbi_server_task_test.exs | 25 +++++++ 9 files changed, 254 insertions(+), 2 deletions(-) create mode 100644 lib/decentralised_book_index/sync/data_transformers/dbi_server_transformer.ex create mode 100644 lib/decentralised_book_index/sync/sync/dbi_server_sync.ex create mode 100644 lib/decentralised_book_index/sync/sync_tasks/sync_dbi_server_task.ex create mode 100644 test/decentralised_book_index/sync/data_transformers/dbi_server_transformer_test.exs create mode 100644 test/decentralised_book_index/sync/sync/dbi_server_sync_test.exs create mode 100644 test/decentralised_book_index/sync/sync_tasks/sync_dbi_server_task_test.exs diff --git a/lib/decentralised_book_index/metadata.ex b/lib/decentralised_book_index/metadata.ex index 9d704f2..085e01d 100644 --- a/lib/decentralised_book_index/metadata.ex +++ b/lib/decentralised_book_index/metadata.ex @@ -102,7 +102,7 @@ defmodule DecentralisedBookIndex.Metadata do end resource DecentralisedBookIndex.Metadata.DBIServer do - define :create_dbi_server, action: :create + define :create_dbi_server, args: [:name, :url, :sync_on?], action: :create define :list_dbi_server, action: :read define :get_dbi_server_by_id, args: [:id], action: :by_id define :search_dbi_server, action: :search, args: [:name] diff --git a/lib/decentralised_book_index/metadata/dbi_server.ex b/lib/decentralised_book_index/metadata/dbi_server.ex index ab4a0ba..1c5850b 100644 --- a/lib/decentralised_book_index/metadata/dbi_server.ex +++ b/lib/decentralised_book_index/metadata/dbi_server.ex @@ -29,11 +29,19 @@ defmodule DecentralisedBookIndex.Metadata.DBIServer do accept [:name, :url, :sync_on?] end + create :sync_create do + accept [:id, :name, :url, :inserted_at, :updated_at, :dbi_server_id] + end + update :update do primary? true accept [:name, :url, :sync_on?] end + update :sync do + accept [:name, :url, :inserted_at, :updated_at, :dbi_server_id] + end + read :by_id do argument :id, :uuid, allow_nil?: false get? true @@ -63,7 +71,7 @@ defmodule DecentralisedBookIndex.Metadata.DBIServer do end attributes do - uuid_primary_key :id + uuid_primary_key :id, writable?: true attribute :name, :string do allow_nil? false diff --git a/lib/decentralised_book_index/sync/data_transformers/dbi_server_transformer.ex b/lib/decentralised_book_index/sync/data_transformers/dbi_server_transformer.ex new file mode 100644 index 0000000..8061bf0 --- /dev/null +++ b/lib/decentralised_book_index/sync/data_transformers/dbi_server_transformer.ex @@ -0,0 +1,21 @@ +defmodule DecentralisedBookIndex.Sync.DataTransformers.DBIServerTransformer do + def from_json(json_body) do + json_body = + if Map.has_key?(json_body, "data") do + json_body["data"] + else + json_body + end + + attrs = + %{ + id: get_in(json_body, ["id"]), + name: get_in(json_body, ["attributes", "name"]), + url: get_in(json_body, ["attributes", "url"]), + inserted_at: get_in(json_body, ["attributes", "inserted_at"]), + updated_at: get_in(json_body, ["attributes", "updated_at"]) + } + + {:ok, attrs} + end +end diff --git a/lib/decentralised_book_index/sync/sync/dbi_server_sync.ex b/lib/decentralised_book_index/sync/sync/dbi_server_sync.ex new file mode 100644 index 0000000..e13eaa1 --- /dev/null +++ b/lib/decentralised_book_index/sync/sync/dbi_server_sync.ex @@ -0,0 +1,31 @@ +defmodule DecentralisedBookIndex.Sync.DBIServerSync do + alias DecentralisedBookIndex.Metadata + alias DecentralisedBookIndex.Metadata.DBIServer + + def create_update(attrs, server_id) do + case Metadata.get_dbi_server_by_id(attrs.id) do + {:ok, dbi_server} -> + attrs = + attrs + |> Map.delete(:id) + |> Map.put(:dbi_server_id, server_id) + + dbi_server + |> Ash.Changeset.for_update(:sync, attrs) + |> Ash.update!(authorize?: false) + + :ok + + {:error, %Ash.Error.Query.NotFound{}} -> + attrs = + attrs + |> Map.put(:dbi_server_id, server_id) + + DBIServer + |> Ash.Changeset.for_create(:sync_create, attrs) + |> Ash.create!(authorize?: false) + + :ok + end + end +end diff --git a/lib/decentralised_book_index/sync/sync_tasks/sync_dbi_server_task.ex b/lib/decentralised_book_index/sync/sync_tasks/sync_dbi_server_task.ex new file mode 100644 index 0000000..ae54ed2 --- /dev/null +++ b/lib/decentralised_book_index/sync/sync_tasks/sync_dbi_server_task.ex @@ -0,0 +1,36 @@ +defmodule DecentralisedBookIndex.SyncTasks.SyncDBIServerTask do + alias DecentralisedBookIndex.Sync.ApiClients.FetchJsons + alias DecentralisedBookIndex.Sync.DataTransformers.DBIServerTransformer + alias DecentralisedBookIndex.Sync.DBIServerSync + + alias DecentralisedBookIndex.Metadata.DBIServer + + require Logger + + def sync(%DBIServer{} = server, url_params \\ "") do + url = "#{server.url}/api/v1/json/servers#{url_params}" + FetchJsons.get(url, sync_closure(server)) + + server + end + + def sync_chunk(json_chunk, server_id) do + for json <- json_chunk do + with {:ok, attrs} <- DBIServerTransformer.from_json(json), + :ok <- DBIServerSync.create_update(attrs, server_id) do + :ok + else + {:error, reason} -> + Logger.error("Pipeline error: #{inspect(reason)}") + end + end + + [] + end + + def sync_closure(server) do + fn json_chunk -> + sync_chunk(json_chunk, server.id) + end + end +end diff --git a/lib/decentralised_book_index/sync/sync_tasks/sync_server_task.ex b/lib/decentralised_book_index/sync/sync_tasks/sync_server_task.ex index 66f5ee7..8fdf936 100644 --- a/lib/decentralised_book_index/sync/sync_tasks/sync_server_task.ex +++ b/lib/decentralised_book_index/sync/sync_tasks/sync_server_task.ex @@ -2,6 +2,7 @@ defmodule DecentralisedBookIndex.SyncTasks.SyncServerTask do alias DecentralisedBookIndex.Metadata alias DecentralisedBookIndex.Metadata.DBIServer + alias DecentralisedBookIndex.SyncTasks.SyncDBIServerTask alias DecentralisedBookIndex.SyncTasks.SyncAuthorsTask alias DecentralisedBookIndex.SyncTasks.SyncPublishersTask alias DecentralisedBookIndex.SyncTasks.SyncBooksTask @@ -20,6 +21,7 @@ defmodule DecentralisedBookIndex.SyncTasks.SyncServerTask do def sync_one(%DBIServer{} = server, url_params \\ "") do server + |> SyncDBIServerTask.sync(url_params) |> SyncAuthorsTask.sync(url_params) |> SyncPublishersTask.sync(url_params) |> SyncBooksTask.sync(url_params) diff --git a/test/decentralised_book_index/sync/data_transformers/dbi_server_transformer_test.exs b/test/decentralised_book_index/sync/data_transformers/dbi_server_transformer_test.exs new file mode 100644 index 0000000..1a06ce9 --- /dev/null +++ b/test/decentralised_book_index/sync/data_transformers/dbi_server_transformer_test.exs @@ -0,0 +1,67 @@ +defmodule DecentralisedBookIndex.Sync.DataTransformers.DBIServerTransformerTest do + use ExUnit.Case, async: true + + alias DecentralisedBookIndex.Sync.DataTransformers.DBIServerTransformer + + describe "correct transformations" do + test "a json contains correct author information" do + json_body = %{ + "data" => %{ + "attributes" => %{ + "inserted_at" => "2025-03-22T20:07:30.766249Z", + "name" => "Test", + "updated_at" => "2025-05-04T18:48:44.213309Z", + "url" => "http://localhost:4001" + }, + "id" => "0c0647ec-07ef-4caa-b683-5847dbfbe5cc", + "links" => %{}, + "meta" => %{}, + "relationships" => %{}, + "type" => "dbi_server" + }, + "jsonapi" => %{"version" => "1.0"}, + "links" => %{ + "self" => + "http://localhost:4000/api/v1/json/servers/0c0647ec-07ef-4caa-b683-5847dbfbe5cc" + }, + "meta" => %{} + } + + assert {:ok, server} = DBIServerTransformer.from_json(json_body) + + assert %{ + id: "0c0647ec-07ef-4caa-b683-5847dbfbe5cc", + name: "Test", + url: "http://localhost:4001", + inserted_at: "2025-03-22T20:07:30.766249Z", + updated_at: "2025-05-04T18:48:44.213309Z" + } = server + end + + test "a json doesn't contains server information \"data\" attribute" do + json_body = %{ + "attributes" => %{ + "inserted_at" => "2025-03-22T20:07:30.766249Z", + "name" => "Test", + "updated_at" => "2025-05-04T18:48:44.213309Z", + "url" => "http://localhost:4001" + }, + "id" => "0c0647ec-07ef-4caa-b683-5847dbfbe5cc", + "links" => %{}, + "meta" => %{}, + "relationships" => %{}, + "type" => "dbi_server" + } + + assert {:ok, server} = DBIServerTransformer.from_json(json_body) + + assert %{ + id: "0c0647ec-07ef-4caa-b683-5847dbfbe5cc", + name: "Test", + url: "http://localhost:4001", + inserted_at: "2025-03-22T20:07:30.766249Z", + updated_at: "2025-05-04T18:48:44.213309Z" + } = server + end + end +end diff --git a/test/decentralised_book_index/sync/sync/dbi_server_sync_test.exs b/test/decentralised_book_index/sync/sync/dbi_server_sync_test.exs new file mode 100644 index 0000000..9ffe237 --- /dev/null +++ b/test/decentralised_book_index/sync/sync/dbi_server_sync_test.exs @@ -0,0 +1,62 @@ +defmodule DecentralisedBookIndex.Sync.DataTransformers.DBIServerSyncTest do + use DecentralisedBookIndex.DataCase, async: true + + alias DecentralisedBookIndex.Sync.DBIServerSync + alias DecentralisedBookIndex.Metadata + + alias DecentralisedBookIndex.TestEndpoints + @test_server_endpoint TestEndpoints.test_api_endpoint() + + setup do + user = generate(user(role: :admin)) + %{user: user} + end + + describe "sync dbi_server transformations" do + test "a new dbi_server will be created" do + server = generate(dbi_server(url: @test_server_endpoint)) + + dbi_server = %{ + id: "0c0647ec-07ef-4caa-b683-5847dbfbe5cc", + name: "Test", + url: "http://localhost:4001", + inserted_at: "2025-03-22T20:07:30.766249Z", + updated_at: "2025-05-04T18:48:44.213309Z" + } + + {:ok, inserted_at, 0} = DateTime.from_iso8601(dbi_server[:inserted_at]) + {:ok, updated_at, 0} = DateTime.from_iso8601(dbi_server[:updated_at]) + + assert :ok = DBIServerSync.create_update(dbi_server, server.id) + assert {:ok, saved_dbi_server} = Metadata.get_dbi_server_by_id(dbi_server.id) + + dbi_server = + dbi_server + |> Map.replace(:inserted_at, inserted_at) + |> Map.replace(:updated_at, updated_at) + + assert dbi_server = saved_dbi_server + assert server.id == saved_dbi_server.dbi_server_id + end + + test "update an existing dbi_server", %{user: user} do + server = generate(dbi_server(url: @test_server_endpoint)) + + {:ok, dbi_server} = Metadata.create_dbi_server("Test", "http://localhost:4001", false, actor: user) + + dbi_server_attrs = %{ + id: dbi_server.id, + name: "Test", + url: "http://localhost:4001", + inserted_at: "2025-03-22T20:07:30.766249Z", + updated_at: "2025-05-04T18:48:44.213309Z" + } + + assert :ok = DBIServerSync.create_update(dbi_server_attrs, server.id) + assert {:ok, saved_dbi_server} = Metadata.get_dbi_server_by_id(dbi_server.id) + + assert dbi_server_attrs = saved_dbi_server + assert server.id == saved_dbi_server.dbi_server_id + end + end +end diff --git a/test/decentralised_book_index/sync/sync_tasks/sync_dbi_server_task_test.exs b/test/decentralised_book_index/sync/sync_tasks/sync_dbi_server_task_test.exs new file mode 100644 index 0000000..009cc69 --- /dev/null +++ b/test/decentralised_book_index/sync/sync_tasks/sync_dbi_server_task_test.exs @@ -0,0 +1,25 @@ +defmodule DecentralisedBookIndex.SyncTasks.SyncDBIServerTaskTest do + use DecentralisedBookIndex.DataCase + + alias DecentralisedBookIndex.SyncTasks.SyncDBIServerTask + alias DecentralisedBookIndex.Metadata + + alias DecentralisedBookIndex.TestEndpoints + @test_server_endpoint TestEndpoints.test_api_endpoint() + + setup do + user = generate(user(role: :admin)) + %{user: user} + end + + describe "sync DBIServer tasks" do + test "sync server", %{user: user} do + server = generate(dbi_server(url: @test_server_endpoint)) + + {:ok, _dbi_server} = Metadata.create_dbi_server("Test", "http://localhost:4001", false, actor: user) + {:ok, _dbi_server} = Metadata.create_dbi_server("Test2", "http://localhost:4001", false, actor: user) + + assert server = SyncDBIServerTask.sync(server) + end + end +end