From 730b2b69271d5afd7206b7a264a1e4d10f28e836 Mon Sep 17 00:00:00 2001 From: KKlochko Date: Wed, 2 Apr 2025 12:54:48 +0300 Subject: [PATCH] Add the sync task for Book. --- lib/decentralised_book_index/metadata/book.ex | 85 ++++++++++- .../data_transformers/book_transformer.ex | 29 ++++ .../sync/sync/book_sync.ex | 31 ++++ .../sync/sync_tasks/sync_books_task.ex | 36 +++++ .../book_transformer_test.exs | 135 ++++++++++++++++++ .../sync/sync/book_sync_test.exs | 70 +++++++++ .../sync/sync_tasks/sync_books_task_test.exs | 20 +++ 7 files changed, 403 insertions(+), 3 deletions(-) create mode 100644 lib/decentralised_book_index/sync/data_transformers/book_transformer.ex create mode 100644 lib/decentralised_book_index/sync/sync/book_sync.ex create mode 100644 lib/decentralised_book_index/sync/sync_tasks/sync_books_task.ex create mode 100644 test/decentralised_book_index/sync/data_transformers/book_transformer_test.exs create mode 100644 test/decentralised_book_index/sync/sync/book_sync_test.exs create mode 100644 test/decentralised_book_index/sync/sync_tasks/sync_books_task_test.exs diff --git a/lib/decentralised_book_index/metadata/book.ex b/lib/decentralised_book_index/metadata/book.ex index 57276d8..49656da 100644 --- a/lib/decentralised_book_index/metadata/book.ex +++ b/lib/decentralised_book_index/metadata/book.ex @@ -54,6 +54,42 @@ defmodule DecentralisedBookIndex.Metadata.Book do change manage_relationship(:author_roles, type: :direct_control, order_is_key: :order) end + create :sync_create do + accept [ + :id, + :title, + :description, + :format, + :language, + :page_count, + :published, + :publisher_id, + :cover_image_url, + :book_editions_registry_id, + :inserted_at, + :updated_at, + :dbi_server_id + ] + + argument :bids, {:array, :map} + argument :author_roles, {:array, :map} + + change fn changeset, _ -> + registry_id = Ash.Changeset.get_attribute(changeset, :book_editions_registry_id) + + if registry_id == nil do + {:ok, registry} = DecentralisedBookIndex.Metadata.create_book_editions_registry() + + Ash.Changeset.force_change_attribute(changeset, :book_editions_registry_id, registry.id) + else + changeset + end + end + + change manage_relationship(:bids, type: :direct_control, order_is_key: :order) + change manage_relationship(:author_roles, type: :direct_control, order_is_key: :order) + end + create :add_book_to_related_editions_registry do accept [ :title, @@ -179,13 +215,51 @@ defmodule DecentralisedBookIndex.Metadata.Book do change manage_relationship(:author_roles, type: :direct_control, order_is_key: :order) end + update :sync do + require_atomic? false + + accept [ + :id, + :title, + :description, + :format, + :language, + :page_count, + :published, + :publisher_id, + :cover_image_url, + :book_editions_registry_id, + :inserted_at, + :updated_at, + :dbi_server_id + ] + + argument :bids, {:array, :map} + argument :author_roles, {:array, :map} + + change fn changeset, _ -> + registry_id = Ash.Changeset.get_attribute(changeset, :book_editions_registry_id) + + if registry_id == nil do + {:ok, registry} = DecentralisedBookIndex.Metadata.create_book_editions_registry() + + Ash.Changeset.force_change_attribute(changeset, :book_editions_registry_id, registry.id) + else + changeset + end + end + + change manage_relationship(:bids, type: :direct_control, order_is_key: :order) + change manage_relationship(:author_roles, type: :direct_control, order_is_key: :order) + end + update :assign_cover_image do accept [:cover_image_url] end end attributes do - uuid_primary_key :id + uuid_primary_key :id, writable?: true attribute :title, :string do allow_nil? false @@ -222,7 +296,10 @@ defmodule DecentralisedBookIndex.Metadata.Book do public? true end - timestamps() + timestamps() do + writable? true + public? true + end end relationships do @@ -230,7 +307,9 @@ defmodule DecentralisedBookIndex.Metadata.Book do belongs_to :book_editions_registry, Metadata.BookEditionsRegistry - belongs_to :publisher, Metadata.Publisher + belongs_to :publisher, Metadata.Publisher do + public? true + end has_many :author_roles, Metadata.AuthorRole do sort order: :asc diff --git a/lib/decentralised_book_index/sync/data_transformers/book_transformer.ex b/lib/decentralised_book_index/sync/data_transformers/book_transformer.ex new file mode 100644 index 0000000..105c404 --- /dev/null +++ b/lib/decentralised_book_index/sync/data_transformers/book_transformer.ex @@ -0,0 +1,29 @@ +defmodule DecentralisedBookIndex.Sync.DataTransformers.BookTransformer do + def from_json(json_body) do + json_body = + if Map.has_key?(json_body, "data") do + json_body["data"] + else + json_body + end + + attrs = + %{ + id: get_in(json_body, ["id"]), + title: get_in(json_body, ["attributes", "title"]), + description: get_in(json_body, ["attributes", "description"]), + cover_image_url: get_in(json_body, ["attributes", "cover_image_url"]), + format: get_in(json_body, ["attributes", "format"]), + language: get_in(json_body, ["attributes", "language"]), + published: get_in(json_body, ["attributes", "published"]), + page_count: get_in(json_body, ["attributes", "page_count"]), + publisher_id: get_in(json_body, ["attributes", "publisher_id"]), + inserted_at: get_in(json_body, ["attributes", "inserted_at"]), + updated_at: get_in(json_body, ["attributes", "updated_at"]), + # relationship + publisher_id: get_in(json_body, ["attributes", "publisher_id"]) + } + + {:ok, attrs} + end +end diff --git a/lib/decentralised_book_index/sync/sync/book_sync.ex b/lib/decentralised_book_index/sync/sync/book_sync.ex new file mode 100644 index 0000000..74938f2 --- /dev/null +++ b/lib/decentralised_book_index/sync/sync/book_sync.ex @@ -0,0 +1,31 @@ +defmodule DecentralisedBookIndex.Sync.BookSync do + alias DecentralisedBookIndex.Metadata + alias DecentralisedBookIndex.Metadata.Book + + def create_update(attrs, server_id) do + case Metadata.get_book_by_id(attrs.id) do + {:ok, book} -> + attrs = + attrs + |> Map.delete(:id) + |> Map.delete(:book_editions_registry) + |> Map.put(:dbi_server_id, server_id) + + book + |> Ash.Changeset.for_update(:sync, attrs) + |> Ash.update!() + + :ok + {:error, %Ash.Error.Query.NotFound{}} -> + attrs = + attrs + |> Map.put(:dbi_server_id, server_id) + + Book + |> Ash.Changeset.for_create(:sync_create, attrs) + |> Ash.create!() + + :ok + end + end +end diff --git a/lib/decentralised_book_index/sync/sync_tasks/sync_books_task.ex b/lib/decentralised_book_index/sync/sync_tasks/sync_books_task.ex new file mode 100644 index 0000000..43ffda9 --- /dev/null +++ b/lib/decentralised_book_index/sync/sync_tasks/sync_books_task.ex @@ -0,0 +1,36 @@ +defmodule DecentralisedBookIndex.SyncTasks.SyncBooksTask do + alias DecentralisedBookIndex.Sync.ApiClients.FetchJsons + alias DecentralisedBookIndex.Sync.DataTransformers.BookTransformer + alias DecentralisedBookIndex.Sync.BookSync + + alias DecentralisedBookIndex.Metadata.DBIServer + + require Logger + + def sync(%DBIServer{} = server) do + url = "#{server.url}/api/v1/json/books" + FetchJsons.get(url, sync_author_closure(server)) + + server + end + + def sync_author_chunk(json_chunk, server_id) do + for json <- json_chunk do + with {:ok, attrs} <- BookTransformer.from_json(json), + :ok <- BookSync.create_update(attrs, server_id) do + :ok + else + {:error, reason} -> + Logger.error("Pipeline error: #{inspect(reason)}") + end + end + + [] + end + + def sync_author_closure(server) do + fn json_chunk -> + sync_author_chunk(json_chunk, server.id) + end + end +end diff --git a/test/decentralised_book_index/sync/data_transformers/book_transformer_test.exs b/test/decentralised_book_index/sync/data_transformers/book_transformer_test.exs new file mode 100644 index 0000000..320fc56 --- /dev/null +++ b/test/decentralised_book_index/sync/data_transformers/book_transformer_test.exs @@ -0,0 +1,135 @@ +defmodule DecentralisedBookIndex.Sync.DataTransformers.BookTransformerTest do + use ExUnit.Case, async: true + + alias DecentralisedBookIndex.Sync.DataTransformers.BookTransformer + alias DecentralisedBookIndex.Metadata.Book + + describe "correct transformations" do + test "a json contains correct book information" do + json_body = %{ + "data" => %{ + "attributes" => %{ + "cover_image_url" => "/images/book_cover.png", + "description" => "A cool book.", + "format" => "Paper", + "inserted_at" => "2025-03-20T14:44:36.162986Z", + "language" => "English", + "page_count" => 1000, + "published" => "2025-03-05", + "publisher_id" => "11349865-1b7b-454a-b999-6c4059888a78", + "title" => "Book", + "updated_at" => "2025-04-01T18:14:25.754055Z" + }, + "id" => "1bbe8861-9d9d-4684-bda6-b6ec238d8d08", + "links" => %{}, + "meta" => %{}, + "relationships" => %{ + "author_roles" => %{ + "links" => %{ + "related" => + "http://localhost:4000/api/v1/json/books/1bbe8861-9d9d-4684-bda6-b6ec238d8d08/author_roles" + }, + "meta" => %{} + }, + "bids" => %{ + "links" => %{ + "related" => + "http://localhost:4000/api/v1/json/books/1bbe8861-9d9d-4684-bda6-b6ec238d8d08/bids" + }, + "meta" => %{} + }, + "publisher" => %{ + "links" => %{ + "related" => + "http://localhost:4000/api/v1/json/books/1bbe8861-9d9d-4684-bda6-b6ec238d8d08/publisher" + }, + "meta" => %{} + } + }, + "type" => "book" + }, + "jsonapi" => %{"version" => "1.0"}, + "links" => %{ + "self" => "http://localhost:4000/api/v1/json/books/1bbe8861-9d9d-4684-bda6-b6ec238d8d08" + }, + "meta" => %{} + } + + assert {:ok, book} = BookTransformer.from_json(json_body) + + assert %{ + id: "1bbe8861-9d9d-4684-bda6-b6ec238d8d08", + cover_image_url: "/images/book_cover.png", + description: "A cool book.", + format: "Paper", + inserted_at: "2025-03-20T14:44:36.162986Z", + language: "English", + page_count: 1000, + published: "2025-03-05", + publisher_id: "11349865-1b7b-454a-b999-6c4059888a78", + title: "Book", + updated_at: "2025-04-01T18:14:25.754055Z" + } = book + end + + test "a json doesn't contains book information \"data\" attribute" do + json_body = %{ + "attributes" => %{ + "cover_image_url" => "/images/book_cover.png", + "description" => "A cool book.", + "format" => "Paper", + "inserted_at" => "2025-03-20T14:44:36.162986Z", + "language" => "English", + "page_count" => 1000, + "published" => "2025-03-05", + "publisher_id" => "11349865-1b7b-454a-b999-6c4059888a78", + "title" => "Book", + "updated_at" => "2025-04-01T18:14:25.754055Z" + }, + "id" => "1bbe8861-9d9d-4684-bda6-b6ec238d8d08", + "links" => %{}, + "meta" => %{}, + "relationships" => %{ + "author_roles" => %{ + "links" => %{ + "related" => + "http://localhost:4000/api/v1/json/books/1bbe8861-9d9d-4684-bda6-b6ec238d8d08/author_roles" + }, + "meta" => %{} + }, + "bids" => %{ + "links" => %{ + "related" => + "http://localhost:4000/api/v1/json/books/1bbe8861-9d9d-4684-bda6-b6ec238d8d08/bids" + }, + "meta" => %{} + }, + "publisher" => %{ + "links" => %{ + "related" => + "http://localhost:4000/api/v1/json/books/1bbe8861-9d9d-4684-bda6-b6ec238d8d08/publisher" + }, + "meta" => %{} + } + }, + "type" => "book" + } + + assert {:ok, book} = BookTransformer.from_json(json_body) + + assert %{ + id: "1bbe8861-9d9d-4684-bda6-b6ec238d8d08", + cover_image_url: "/images/book_cover.png", + description: "A cool book.", + format: "Paper", + inserted_at: "2025-03-20T14:44:36.162986Z", + language: "English", + page_count: 1000, + published: "2025-03-05", + publisher_id: "11349865-1b7b-454a-b999-6c4059888a78", + title: "Book", + updated_at: "2025-04-01T18:14:25.754055Z" + } = book + end + end +end diff --git a/test/decentralised_book_index/sync/sync/book_sync_test.exs b/test/decentralised_book_index/sync/sync/book_sync_test.exs new file mode 100644 index 0000000..caf9780 --- /dev/null +++ b/test/decentralised_book_index/sync/sync/book_sync_test.exs @@ -0,0 +1,70 @@ +defmodule DecentralisedBookIndex.Sync.DataTransformers.BookSyncTest do + use DecentralisedBookIndex.DataCase, async: true + + alias DecentralisedBookIndex.Sync.BookSync + alias DecentralisedBookIndex.Metadata + + alias DecentralisedBookIndex.TestEndpoints + @test_server_endpoint TestEndpoints.test_api_endpoint() + + describe "sync book transformations" do + test "a new book will be created" do + server = generate(dbi_server(url: @test_server_endpoint)) + + publisher = generate(publisher()) + + book = %{ + id: "1bbe8861-9d9d-4684-bda6-b6ec238d8d08", + cover_image_url: "/images/book_cover.png", + description: "A cool book.", + format: "Paper", + inserted_at: "2025-03-20T14:44:36.162986Z", + language: "English", + page_count: 1000, + published: "2025-03-05", + publisher_id: publisher.id, + title: "Book", + updated_at: "2025-04-01T18:14:25.754055Z" + } + + {:ok, inserted_at, 0} = DateTime.from_iso8601(book[:inserted_at]) + {:ok, updated_at, 0} = DateTime.from_iso8601(book[:updated_at]) + + assert :ok = BookSync.create_update(book, server.id) + assert {:ok, saved_book} = Metadata.get_book_by_id(book.id) + + book = + book + |> Map.replace(:inserted_at, inserted_at) + |> Map.replace(:updated_at, updated_at) + + assert book = saved_book + assert nil != saved_book.book_editions_registry_id + assert server.id == saved_book.dbi_server_id + end + + test "update an existing book" do + server = generate(dbi_server(url: @test_server_endpoint)) + + book = generate(book()) + + book_attrs = %{ + id: book.id, + cover_image_url: "/images/book_cover2.png", + description: "A cool book 2.", + format: "Ebook", + inserted_at: "2025-01-20T14:44:36.162986Z", + language: "English", + page_count: 1001, + published: "2025-03-05", + title: "Book2", + updated_at: "2025-02-01T18:14:25.754055Z" + } + + assert :ok = BookSync.create_update(book_attrs, server.id) + assert {:ok, saved_book} = Metadata.get_book_by_id(book.id) + + assert book = saved_book + end + end +end diff --git a/test/decentralised_book_index/sync/sync_tasks/sync_books_task_test.exs b/test/decentralised_book_index/sync/sync_tasks/sync_books_task_test.exs new file mode 100644 index 0000000..527a7cd --- /dev/null +++ b/test/decentralised_book_index/sync/sync_tasks/sync_books_task_test.exs @@ -0,0 +1,20 @@ +defmodule DecentralisedBookIndex.SyncTasks.SyncBookTaskTest do + use DecentralisedBookIndex.DataCase + + alias DecentralisedBookIndex.SyncTasks.SyncBooksTask + alias DecentralisedBookIndex.Metadata + + alias DecentralisedBookIndex.TestEndpoints + @test_server_endpoint TestEndpoints.test_api_endpoint() + + describe "sync authors tasks" do + test "sync authors" do + server = generate(dbi_server(url: @test_server_endpoint)) + + _book = generate(book()) + _book = generate(book()) + + assert server = SyncBooksTask.sync(server) + end + end +end