diff --git a/benchmarks/qlc_bench.exs b/benchmarks/qlc_bench.exs deleted file mode 100644 index c7bf8c3c2..000000000 --- a/benchmarks/qlc_bench.exs +++ /dev/null @@ -1,161 +0,0 @@ -## qlc_bench -# -## PURPOSE -# -# This module benchmarks various ways of dealing with QLC tables. It is mainly -# aimed to gather performance insights about how to perform QLC queries in -# nostrum, and aims to potentially also propose improvements upstream, if -# applicable. -# -# Note that if using the QLC cursor functionality, memory measurements are not -# representative! QLC's cursor sets up a new process that is queried, and -# Benchee cannot measure that. -# -## SUPPORTING FILES -# -# This module requires that you put the following module into -# `src/qlctest.erl`: < -# qlc:q([{Id, Id, Value} || {Id, Value} <- Tab, Id =:= RequestedId]). -# -# qh1b(Tab, RequestedId) -> -# V1 = qlc:q([{Id, Id, Value} || {Id, Value} <- Tab]), -# qlc:q([Value || {Id, _, Value} <- V1, Id =:= RequestedId]). -# -# qh2a(Tab, RequestedId) -> -# qlc:q([{Id, Id, Value} || {Id, _, Value} <- Tab, Id =:= RequestedId]). -# -# EOF -# -## FINDINGS -# -# Due to (presumably) the QLC parse transform compile time query optimizations, -# query list comprehensions written directly in Erlang can achieve better -# resource usage than those written in Elixir. - -tab = :ets.new(:test, [:set, :public]) -:ets.insert(tab, Enum.map(1..100_000, &{&1, %{id: &1, name: "#{&1}"}})) - -IO.puts( - "Table size is: #{:ets.info(tab, :memory) * :erlang.system_info(:wordsize) / 1024 / 1024} MB" -) - -qh = :ets.table(tab) -ms = [{{:"$1", :"$2"}, [], [{{:"$1", :"$1", :"$2"}}]}] -qhtraverse = :ets.table(tab, traverse: {:select, ms}) - -# Elixir queries -qh0 = - :qlc.string_to_handle(~c"[{Id, Id, Value} || {Id, Value} <- Handle, Id =:= RequestedId].", [], - Handle: qh, - RequestedId: 500_000 - ) - -qh1a = - :qlc.string_to_handle(~c"[{Id, Id, Value} || {Id, Value} <- Handle].", [], - Handle: qh, - RequestedId: 500_000 - ) - -qh1b = - :qlc.string_to_handle(~c"[Value || {Id, _, Value} <- Handle, Id =:= RequestedId].", [], - Handle: qh1a, - RequestedId: 500_000 - ) - -qh2a = - :qlc.string_to_handle(~c"[{Id, Id, Value} || {Id, Value} <- Handle, Id =:= RequestedId].", [], - Handle: qhtraverse, - RequestedId: 500_000 - ) - -# Erlang queries -nqh0 = :qlctest.qh0(qh, 500_000) -nqh1b = :qlctest.qh1b(qh, 500_000) -nqh2a = :qlctest.qh2a(qhtraverse, 500_000) - -# Print query information. -# These _should_ be the same between Elixir and Erlang. -# IO.puts("===== NAIVE (Elixir) =====") -# IO.puts(:qlc.info(qh0)) -# IO.puts("===== NAIVE (Erlang) =====") -# IO.puts(:qlc.info(nqh0)) -# IO.puts("===== NESTED (Elixir) =====") -# IO.puts(:qlc.info(qh1b)) -# IO.puts("===== NESTED (Erlang) =====") -# IO.puts(:qlc.info(nqh1b)) -# IO.puts("===== NAIVE, TRAVERSED (Elixir) =====") -# IO.puts(:qlc.info(qh2a)) -# IO.puts("===== NAIVE, TRAVERSED (Erlang) =====") -# IO.puts(:qlc.info(nqh2a)) - -counter = fn _item, acc -> acc + 1 end - -_cursor_counter = fn qh -> - Stream.resource( - fn -> :qlc.cursor(qh) end, - fn cursor -> - # With the default of 10, this takes forever. - case :qlc.next_answers(cursor, 500) do - results -> {[Enum.count(results)], cursor} - [] -> {:halt, cursor} - end - end, - fn cursor -> :qlc.delete_cursor(cursor) end - ) -end - -must! = fn - [] -> :ok - [_] -> :ok - num when is_integer(num) -> :ok -end - -Benchee.run( - %{ - # Queries coming from `string_to_handle` in Elixir. - "naive" => fn -> must!.(:qlc.e(qh0)) end, - "nested" => fn -> must!.(:qlc.e(qh1b)) end, - "naive, traversed" => fn -> must!.(:qlc.e(qh2a)) end, - "naive, fold" => fn -> must!.(:qlc.fold(counter, 0, qh0)) end, - "nested, fold" => fn -> must!.(:qlc.fold(counter, 0, qh1b)) end, - "naive, traversed, fold" => fn -> must!.(:qlc.fold(counter, 0, qh2a)) end, - - # No effect. - # "naive, fold, uncached" => fn -> :qlc.fold(counter, 0, qh0, cache_all: false) end, - # "nested, fold, uncached" => fn -> :qlc.fold(counter, 0, qh1b, cache_all: false) end, - # "naive, traversed, fold, uncached" => fn -> :qlc.fold(counter, 0, qh2a, cache_all: false) end, - - # A lot slower. - # "naive, cursor" => fn -> cursor_counter.(qh0) |> Enum.sum() end, - # "nested, cursor" => fn -> cursor_counter.(qh1b) |> Enum.sum() end, - # "naive, traversed, cursor" => fn -> cursor_counter.(qh2a) |> Enum.sum() end, - - # Queries coming from the parse transform in Erlang. - "native, naive" => fn -> must!.(:qlc.e(nqh0)) end, - "native, nested" => fn -> must!.(:qlc.e(nqh1b)) end, - "native, naive, traversed" => fn -> must!.(:qlc.e(nqh2a)) end, - "native, naive, fold" => fn -> must!.(:qlc.fold(counter, 0, nqh0)) end, - "native, nested, fold" => fn -> must!.(:qlc.fold(counter, 0, nqh1b)) end, - "native, naive, traversed, fold" => fn -> must!.(:qlc.fold(counter, 0, nqh2a)) end - - # No effect. - # "native, naive, fold, uncached" => fn -> :qlc.fold(counter, 0, nqh0, cache_all: false) end, - # "native, nested, fold, uncached" => fn -> :qlc.fold(counter, 0, nqh1b, cache_all: false) end, - # "native, naive, traversed, fold, uncached" => fn -> :qlc.fold(counter, 0, nqh2a, cache_all: false) end, - - # A lot slower. - # "native, naive, cursor" => fn -> cursor_counter.(nqh0) |> Enum.sum() end, - # "native, nested, cursor" => fn -> cursor_counter.(nqh1b) |> Enum.sum() end, - # "native, naive, traversed, cursor" => fn -> cursor_counter.(nqh2a) |> Enum.sum() end - }, - memory_time: 2, - time: 2 -) diff --git a/guides/cheat-sheets/qlc.cheatmd b/guides/cheat-sheets/qlc.cheatmd deleted file mode 100644 index 1dea50643..000000000 --- a/guides/cheat-sheets/qlc.cheatmd +++ /dev/null @@ -1,91 +0,0 @@ -# QLC Usage - -This cheat sheet covers some example queries using Query-List Comprehensions in Erlang, as well as some debugging tips. - -QLC modules must include this library include as part of their prelude: - -```erl --include_lib("stdlib/include/qlc.hrl"). -``` - -As per the Erlang docs for QLC: - -> This causes a parse transform to substitute a fun for the QLC. The (compiled) fun is called when the query handle is evaluated. - -## Examples - -### Fetch role members - -```erl -find_role_users(RequestedGuildId, RoleId, MemberCache) -> - qlc:q([Member || {{GuildId, MemberId}, Member} <- MemberCache:query_handle(), - % Filter to member objects of the selected guild - GuildId =:= RequestedGuildId, - % Filter to members of the provided role - lists:member(RoleId, map_get(roles, Member))]). -``` - -### Fetch guilds over a certain size - -```erl -find_large_communities(Threshold, GuildCache) -> - qlc:q([Guild || {_, Guild} <- GuildCache:query_handle(), - % Filter for guilds that are over the provided size - map_get(member_count, Guild) > Threshold]). -``` - -### Find all online users in a guild - -```erl -find_online_users(RequestedGuildId, PresenceCache, UserCache) -> - qlc:q([User || {{GuildId, PresenceUserId}, Presence} <- PresenceCache:query_handle(), - % Filter to members in the requested guild ID - GuildId =:= RequestedGuildId, - % Fetch any members where the status is not offline - map_get(status, Presence) /= offline, - % Join the found users on the UserCache - {UserId, User} <- UserCache:query_handle(), - % Return the users that match the found presences - UserId =:= PresenceUserId]). -``` - -This depends on the guild presences intent being enabled and the bot storing received presences in the presence cache. - -### Getting the largest N guilds - -```erl -top_guilds(N, GuildCache) -> - Q = qlc:q([{MemberCount, Guild} || {_Id, #{member_count := MemberCount} = Guild} <- GuildCache:query_handle()]), - Q2 = qlc:keysort(1, Q, [{order, descending}]), - GuildCache:wrap_qlc(fun () -> - C = qlc:cursor(Q2), - R = qlc:next_answers(C, N), - ok = qlc:delete_cursor(C), - R - end). -``` - -## Debugging QLC - -{: .col-2 } - -### View query info - -You can use `:qlc.info/1` to evaluate how Erlang will parse a query. You can read the Erlang docs for an explanation of the value returned by this call - -```elixir -:qlc.info( - :my_queries.find_users(..., Nostrum.Cache.UserCache) -) -``` - -### Sampling a query - -You can return only X records that match a query using `:qlc.next_answers/2`, passing in the query as the first argument and the number of answers to return as the second argument. - -```elixir -:qlc.next_answers( - :my_queries.find_users(..., Nostrum.Cache.UserCache), - 5 -) -``` diff --git a/guides/functionality/state.md b/guides/functionality/state.md index ade3dd27e..7c2940181 100644 --- a/guides/functionality/state.md +++ b/guides/functionality/state.md @@ -13,101 +13,17 @@ free to suggest it [on GitHub](https://github.com/Kraigie/nostrum/issues). Should the default ETS-based caching not be enough for you - for instance, you want to integrate to some external caching mechanism or want to distribute your -bot across multiple nodes, please see the [pluggable +bot across multiple nodes, and the built-in Mnesia-based caching is not enough +for you either, please see the [pluggable caching](../advanced/pluggable_caching.md) documentation. -## Query list comprehensions - -nostrum's built-in functions to query the cache should be sufficient to cover -common use cases. If you need more involved queries, it is recommended to use -nostrum's [qlc](https://www.erlang.org/doc/man/qlc.html) support. - -### Examples - -Below you can find some example queries using QLC. - -```erl -% src/nostrum_queries.erl - --module(nostrum_queries). --export([find_role_users/4, find_large_communities/2]). - --include_lib("stdlib/include/qlc.hrl"). - -% Find the Nostrum.Struct.User and Member objects of all members in a specific guild role. -find_role_users(RequestedGuildId, RoleId, MemberCache, UserCache) -> - qlc:q([{User, Member} || {{GuildId, MemberId}, Member} <- MemberCache:query_handle(), - % Filter to member objects of the selected guild - GuildId =:= RequestedGuildId, - % Filter to members of the provided role - lists:member(RoleId, map_get(roles, Member)), - % Get a handle on the UserCache table - {UserId, User} <- UserCache:query_handle(), - % Find the User struct that matches the found Member - MemberId =:= UserId]). - -% Find all communities in the Guild cache with the COMMUNITY guild feature -% that are over a certain threshold in user size -find_large_communities(Threshold, GuildCache) -> - qlc:q([Guild || {_, Guild} <- GuildCache:query_handle(), - % Filter for guilds that are over the provided size - map_get(member_count, Guild) > Threshold, - % Filter for guilds that have COMMUNITY in the features field - lists:member(<<"COMMUNITY">>, map_get(features, Guild))]). -``` - -`nostrum_queries:find_role_users/4` fetches all users in the specified guild -(`RequestedGuildId`) with the role `RoleId`. The code is annotated, but -step-by-step the flow is: the member cache is filtered down to all members in -the guild, then using `lists:member/2` we check for role membership, and finally -we join against the user cache to return full user objects in the result. - -`nostrum_queries:find_large_communities/2` fetches all guilds in the guild cache -that meet the criteria of having at least `Threshold` members *and* having the -`COMMUNITY` guild feature set to true in the Discord UI. It is easy to follow -the flow of this query by the annotations, with only some minor things to note -such as needing to use `<<"bitstring">>` bit syntax to represent the strings, -which is implicit in Elixir. - -In Elixir, you can call these queries like so using `:qlc.eval/1`: - -```elixir -matching_guilds = :qlc.eval(:nostrum_queries.find_large_communities(50, Nostrum.Cache.GuildCache)) -``` - -### Implementing your own queries and caches - -By [implementing a QLC -table](https://www.erlang.org/doc/man/qlc.html#implementing_a_qlc_table), all -read operations from nostrum will be performed over your QLC table -implementation alone, and nostrum's dispatcher modules can easily be expanded -for more queries in the future. If you've never heard of QLC before, the -[`beam-lazy` repository](https://github.com/savonarola/beam-lazy) contains a -good introduction. - -Using QLC bring a plethora of benefits. Implementation of a QLC table is -relatively simple, and gives us compile-time query optimization and compilation -in native Erlang list comprehension syntax. Furthermore, should you wish to -perform queries on your caches beyond what nostrum offers out of the box, you -can write your queries using the `query_handle/0` functions on our caches, -without having to investigate their exact API. - -There is one caveat to be aware of when writing cache adapters in Elixir that -build on this functionality: While Erlang's QLC can perform intelligent query -optimization, a lot of it is implemented via a parse transform and thus only -available at compile time in Erlang modules. It is therefore recommended to -write your QLC queries in Erlang modules: in Mix projects this can be achieved -easily via the `src/` directory. Read the [QLC module -documentation](https://www.erlang.org/doc/man/qlc.html) for more details on the -optimizations done. - -The reason why QLC is being used as opposed to the Elixir-traditional stream API -is that the stream API does not support a number of features we are using here. -Apart from that, nostrum's previous API (`select` and friends) gave users a -false impression that nostrum was doing an efficient iteration under the hood, -which caused issues for large bots. +## Implementing your own caches +To implement custom caches, implement the behaviour defined by the cache +module, such as `Nostrum.Cache.GuildCache`. For ease of use, these modules +define both a user-facing API to obtain objects from the configured cache, as +well as the developer-facing behaviour description. ## Internal state diff --git a/lib/nostrum/cache/guild_cache.ex b/lib/nostrum/cache/guild_cache.ex index 880fd7a79..4d32d90b9 100644 --- a/lib/nostrum/cache/guild_cache.ex +++ b/lib/nostrum/cache/guild_cache.ex @@ -14,7 +14,7 @@ defmodule Nostrum.Cache.GuildCache do ## Writing your own guild cache - As with the other caches, the guild cache API consists of two parts: + As with the other caches, the guild cache API consists of three parts: - The functions that nostrum calls, such as `c:create/1` or `c:update/1`. These **do not create any objects in the Discord API**, they are purely @@ -22,11 +22,12 @@ defmodule Nostrum.Cache.GuildCache do want to create objects on Discord, use the functions exposed by `Nostrum.Api` instead. - - the QLC query handle for read operations, `c:query_handle/0`, and + - functions for read operations, almost exclusively called by the end user - the `c:child_spec/1` callback for starting the cache under a supervisor. - You need to implement all of them for nostrum to work with your custom + You need to implement all callbacks in this module (except for + `c:wrap_query/1`, since it is optional) for nostrum to work with your custom cache. The "upstream data" wording in this module references the fact that the @@ -60,7 +61,7 @@ defmodule Nostrum.Cache.GuildCache do ## Parameters - `acc`: The initial accumulator. Also returned if no guilds are cached. - - `fun`: Called for every guild in the result. Takes a pair in the form + - `reducer`: Called for every guild in the result. Takes a pair in the form `(guild, acc)`, and must return the updated accumulator. - `cache` (optional): The cache to use. nostrum will use the cache configured at compile time by default. @@ -69,8 +70,8 @@ defmodule Nostrum.Cache.GuildCache do @spec fold(acc, (Guild.t(), acc -> acc)) :: acc when acc: term() @spec fold(acc, (Guild.t(), acc -> acc), module()) :: acc when acc: term() def fold(acc, reducer, cache \\ @configured_cache) do - handle = :nostrum_guild_cache_qlc.all(cache) - wrap_qlc(cache, fn -> :qlc.fold(reducer, acc, handle) end) + enumerable = cache.all() + wrap_query(cache, fn -> Enum.reduce(enumerable, acc, reducer) end) end @doc """ @@ -80,6 +81,11 @@ defmodule Nostrum.Cache.GuildCache do """ @callback get(Guild.id()) :: {:ok, Guild.t()} | {:error, :not_found} + @doc """ + Return an enumerable for all guilds in the cache. + """ + @callback all() :: Enumerable.t(Guild.t()) + @doc """ Retrieves a single `Nostrum.Struct.Guild` from the cache via its `id`. @@ -87,6 +93,14 @@ defmodule Nostrum.Cache.GuildCache do """ defdelegate get(guild_id), to: @configured_cache + @doc """ + Returns an enumerable that yields all guilds in the cache. + + You must wrap calls to this function in `wrap_query/1`. + """ + @doc since: "0.10.0" + defdelegate all, to: @configured_cache + # Functions called from nostrum. @doc "Create a guild in the cache." @@ -199,40 +213,18 @@ defmodule Nostrum.Cache.GuildCache do @callback member_count_down(Guild.id()) :: true @doc """ - Return a QLC query handle for cache read operations. - - This is used by nostrum to provide any read operations on the cache. Write - operations still need to be implemented separately. - - The Erlang manual on [Implementing a QLC - Table](https://www.erlang.org/doc/man/qlc.html#implementing_a_qlc_table) - contains examples for implementation. To prevent full table scans, accept - match specifications in your `TraverseFun` and implement a `LookupFun` as - documented. - - The query handle must return items in the form `{guild_id, guild}`, where: - - `guild_id` is a `t:Nostrum.Struct.Guild.id/0`, and - - `guild` is a `t:Nostrum.Struct.Guild.t/0`. - - If your cache needs some form of setup or teardown for QLC queries (such as - opening connections), see `c:wrap_qlc/1`. - """ - @doc since: "0.8.0" - @callback query_handle() :: :qlc.query_handle() - - @doc """ - A function that should wrap any `:qlc` operations. + A function that should wrap any queries. If you implement a cache that is backed by a database and want to perform cleanup and teardown actions such as opening and closing connections, managing transactions and so on, you want to implement this function. nostrum - will then effectively call `wrap_qlc(fn -> :qlc.e(...) end)`. + will then effectively call `wrap_query(fn -> ... end)`. If your cache does not need any wrapping, you can omit this. """ - @doc since: "0.8.0" - @callback wrap_qlc((-> result)) :: result when result: term() - @optional_callbacks wrap_qlc: 1 + @doc since: "0.10.0" + @callback wrap_query((-> result)) :: result when result: term() + @optional_callbacks wrap_query: 1 @doc """ Retrieve the child specification for starting this mapping under a supervisor. @@ -282,24 +274,18 @@ defmodule Nostrum.Cache.GuildCache do end @doc """ - Call `c:wrap_qlc/1` on the given cache, if implemented. + Call `c:wrap_query/1` on the given cache, if implemented. If no cache is given, calls out to the default cache. """ @doc since: "0.8.0" - @spec wrap_qlc((-> result)) :: result when result: term() - @spec wrap_qlc(module(), (-> result)) :: result when result: term() - def wrap_qlc(cache \\ @configured_cache, fun) do - if function_exported?(cache, :wrap_qlc, 1) do - cache.wrap_qlc(fun) + @spec wrap_query((-> result)) :: result when result: term() + @spec wrap_query(module(), (-> result)) :: result when result: term() + def wrap_query(cache \\ @configured_cache, fun) do + if function_exported?(cache, :wrap_query, 1) do + cache.wrap_query(fun) else fun.() end end - - @doc """ - Return the QLC handle of the configured cache. - """ - @doc since: "0.8.0" - defdelegate query_handle(), to: @configured_cache end diff --git a/lib/nostrum/cache/guild_cache/ets.ex b/lib/nostrum/cache/guild_cache/ets.ex index fa5d98e00..ca102a724 100644 --- a/lib/nostrum/cache/guild_cache/ets.ex +++ b/lib/nostrum/cache/guild_cache/ets.ex @@ -56,6 +56,24 @@ defmodule Nostrum.Cache.GuildCache.ETS do end end + @impl GuildCache + @doc since: "0.10.0" + @spec all() :: Enumerable.t(Guild.t()) + def all do + ms = [{{:_, :"$1"}, [], [:"$1"]}] + Stream.resource( + fn -> :ets.select(@table_name, ms, 100) + end, + fn items -> + case items do + {matches, cont} -> + {matches, :ets.select(cont)} + :"$end_of_table" -> {:halt, nil} end + end, + fn _cont -> :ok end + ) + end + @doc "Create the given guild in the cache." @impl GuildCache @spec create(map()) :: Guild.t() @@ -237,12 +255,4 @@ defmodule Nostrum.Cache.GuildCache.ETS do true end end - - @impl GuildCache - @doc "Get a QLC query handle for the guild cache." - @doc since: "0.8.0" - @spec query_handle :: :qlc.query_handle() - def query_handle do - :ets.table(@table_name) - end end diff --git a/lib/nostrum/cache/guild_cache/mnesia.ex b/lib/nostrum/cache/guild_cache/mnesia.ex index e059f667d..d8b7815d0 100644 --- a/lib/nostrum/cache/guild_cache/mnesia.ex +++ b/lib/nostrum/cache/guild_cache/mnesia.ex @@ -69,6 +69,23 @@ if Code.ensure_loaded?(:mnesia) do end) end + @impl GuildCache + @doc since: "0.10.0" + @spec all() :: Enumerable.t(Guild.t()) + def all do + ms = [{{:_, :_, :"$1"}, [], [:"$1"]}] + Stream.resource( + fn -> :mnesia.select(@table_name, ms, 100, :read) end, + fn items -> + case items do + {matches, cont} -> + {matches, :mnesia.select(cont)} + :"$end_of_table" -> {:halt, nil} end + end, + fn _cont -> :ok end + ) + end + # Used by dispatch @impl GuildCache @@ -300,16 +317,8 @@ if Code.ensure_loaded?(:mnesia) do end @impl GuildCache - @doc "Get a QLC handle for the guild cache." - @spec query_handle :: :qlc.query_handle() - def query_handle do - ms = [{{:_, :"$1", :"$2"}, [], [{{:"$1", :"$2"}}]}] - :mnesia.table(@table_name, {:traverse, {:select, ms}}) - end - - @impl GuildCache - @doc "Wrap QLC operations in a transaction." - def wrap_qlc(fun) do + @doc "Wrap queries in a transaction." + def wrap_query(fun) do :mnesia.activity(:sync_transaction, fun) end end diff --git a/lib/nostrum/cache/guild_cache/noop.ex b/lib/nostrum/cache/guild_cache/noop.ex index 7d51d36b4..cf2594cb3 100644 --- a/lib/nostrum/cache/guild_cache/noop.ex +++ b/lib/nostrum/cache/guild_cache/noop.ex @@ -29,6 +29,9 @@ defmodule Nostrum.Cache.GuildCache.NoOp do @impl GuildCache def get(_guild_id), do: {:error, :not_found} + @impl GuildCache + def all, do: [] + @impl GuildCache def create(payload), do: Guild.to_struct(payload) @@ -82,7 +85,4 @@ defmodule Nostrum.Cache.GuildCache.NoOp do @impl GuildCache def member_count_down(_guild_id), do: true - - @impl GuildCache - def query_handle, do: :qlc.string_to_handle(~c"[].") end diff --git a/lib/nostrum/cache/member_cache.ex b/lib/nostrum/cache/member_cache.ex index a3486dca4..51132e725 100644 --- a/lib/nostrum/cache/member_cache.ex +++ b/lib/nostrum/cache/member_cache.ex @@ -27,8 +27,24 @@ defmodule Nostrum.Cache.MemberCache do @doc """ Retrieve a member from the cache by guild and user ID. """ + @doc since: "0.10.0" @callback get(Guild.id(), Member.user_id()) :: {:ok, Member.t()} | {:error, atom()} + @doc """ + Yield an enumerable of `{guild_id, member}` pairs associated with the given user ID. + + Since nostrum does not know when it can stop streaming you results, you have + to wrap calls to this function in `wrap_query/1`. + """ + @doc since: "0.10.0" + @callback by_user(User.id()) :: {:ok, Enumerable.t({Guild.id(), Member.t()})} | {:error. atom()} + + @doc """ + Yield an enumerable of members associated with the given guild ID. + """ + @doc since: "0.10.0" + @callback by_guild(Guild.id()) :: {:ok, Enumerable.t(Member.t())} | {:error. atom()} + @doc """ Add the member for the given guild from upstream data. @@ -70,47 +86,24 @@ defmodule Nostrum.Cache.MemberCache do """ @callback bulk_create(Guild.id(), members :: [member :: map()]) :: true - @doc """ - Return a QLC query handle for cache read operations. - - This is used by nostrum to provide any read operations on the cache. Write - operations still need to be implemented separately. - - The Erlang manual on [Implementing a QLC - Table](https://www.erlang.org/doc/man/qlc.html#implementing_a_qlc_table) - contains examples for implementation. To prevent full table scans, accept - match specifications in your `TraverseFun` and implement a `LookupFun` as - documented. - - The query handle must return items in the form `{guild_id, user_id, - member}`, where: - - `guild_id` is a `t:Nostrum.Struct.Guild.id/0`, - - `user_id` is a `t:Nostrum.Struct.User.id/0`, and - - `member` is a `t:Nostrum.Struct.Guild.Member.t/0`. - - If your cache needs some form of setup or teardown for QLC queries (such as - opening connections), see `c:wrap_qlc/1`. - """ - @callback query_handle() :: :qlc.query_handle() - @doc """ Retrieve the child specification for starting this mapping under a supervisor. """ @callback child_spec(term()) :: Supervisor.child_spec() @doc """ - A function that should wrap any `:qlc` operations. + A function that should wrap any long-running query operations. If you implement a cache that is backed by a database and want to perform cleanup and teardown actions such as opening and closing connections, managing transactions and so on, you want to implement this function. nostrum - will then effectively call `wrap_qlc(fn -> :qlc.e(...) end)`. + will then effectively call `wrap_query(fn -> ... end)`. If your cache does not need any wrapping, you can omit this. """ - @doc since: "0.8.0" - @callback wrap_qlc((-> result)) :: result when result: term() - @optional_callbacks wrap_qlc: 1 + @doc since: "0.10.0" + @callback wrap_query((-> result)) :: result when result: term() + @optional_callbacks wrap_query: 1 # User-facing @@ -141,6 +134,10 @@ defmodule Nostrum.Cache.MemberCache do The members will be returned alongside their guild ID as a pair in the format `{guild_id, member}`. + + ## Raises + + Fails when `by_user/1` of the selected cache returns an error. """ @doc since: "0.8.0" @spec fold_by_user(acc, Member.user_id(), ({Guild.id(), Member.t()}, acc -> acc)) :: acc @@ -149,8 +146,8 @@ defmodule Nostrum.Cache.MemberCache do acc when acc: term() def fold_by_user(acc, user_id, member_reducer, cache \\ @configured_cache) do - handle = :nostrum_member_cache_qlc.by_user(user_id, cache) - wrap_qlc(cache, fn -> :qlc.fold(member_reducer, acc, handle) end) + enumerable = cache.by_user(user_id) + wrap_query(cache, fn -> Enum.reduce(enumerable, acc, member_reducer) end) end @doc """ @@ -166,20 +163,24 @@ defmodule Nostrum.Cache.MemberCache do - `acc`: The initial accumulator. Also returned if no guild members were found. - `guild_id`: The guild for which to reduce members. - - `fun`: Called for every element in the result. Takes a pair + - `member_reducer`: Called for every element in the result. Takes a pair in the form `(member, acc)`, and must return the updated accumulator. ## Return value Returns the resulting accumulator via `fun`. Returns `acc` unchanged if no results were found. + + ## Raises + + Fails when `by_guild/1` of the selected cache returns an error. """ @doc since: "0.8.0" @spec fold(acc, Guild.id(), (Member.t(), acc -> acc)) :: acc when acc: term() @spec fold(acc, Guild.id(), (Member.t(), acc -> acc), module()) :: acc when acc: term() def fold(acc, guild_id, member_reducer, cache \\ @configured_cache) do - handle = :nostrum_member_cache_qlc.by_guild(guild_id, cache) - wrap_qlc(cache, fn -> :qlc.fold(member_reducer, acc, handle) end) + enumerable = cache.by_guild(guild_id) + wrap_query(cache, fn -> Enum.reduce(enumerable, acc, member_reducer) end) end @doc """ @@ -202,6 +203,10 @@ defmodule Nostrum.Cache.MemberCache do If the user for a guild member is not found, the member _and_ user won't be present in the result. Barring a bug in nostrum's caching, this should never happen in practice. + + ## Raises + + Fails when `by_guild/1` of the selected cache returns an error. """ @doc since: "0.8.0" @spec fold_with_users(acc, Guild.id(), ({Member.t(), User.t()}, acc -> acc)) :: acc @@ -210,35 +215,42 @@ defmodule Nostrum.Cache.MemberCache do acc when acc: term() def fold_with_users(acc, guild_id, fun, cache \\ @configured_cache) do - joined_handle = :nostrum_member_cache_qlc.get_with_users(guild_id, cache, UserCache) - - wrapped_fun = fn {member, user}, acc -> - fun.({member, user}, acc) - end - - wrap_qlc(cache, fn -> - UserCache.wrap_qlc(fn -> - :qlc.fold(wrapped_fun, acc, joined_handle) - end) + enumerable = cache.by_guild(guild_id) + + wrap_query(cache, fn -> + Enum.reduce( + enumerable, + acc, + fn (%Member{user_id: user_id} = member, acc) -> + # credo:disable-for-next-line + case UserCache.get(user_id) do + {:ok, user} -> fun.({member, user}, acc) + _error -> acc + end + end + ) end) end @doc """ - Call `c:wrap_qlc/1` on the given cache, if implemented. + Call `c:wrap_query/1` on the given cache, if implemented. If no cache is given, calls out to the default cache. """ - @doc since: "0.8.0" - @spec wrap_qlc((-> result)) :: result when result: term() - @spec wrap_qlc(module(), (-> result)) :: result when result: term() - def wrap_qlc(cache \\ @configured_cache, fun) do - if function_exported?(cache, :wrap_qlc, 1) do - cache.wrap_qlc(fun) + @doc since: "0.10.0" + @spec wrap_query((-> result)) :: result when result: term() + @spec wrap_query(module(), (-> result)) :: result when result: term() + def wrap_query(cache \\ @configured_cache, fun) do + if function_exported?(cache, :wrap_query, 1) do + cache.wrap_query(fun) else fun.() end end + defdelegate by_guild(guild_id), to: @configured_cache + defdelegate by_user(user_id), to: @configured_cache + # Nostrum-facing @doc false defdelegate create(guild_id, member), to: @configured_cache @@ -249,12 +261,6 @@ defmodule Nostrum.Cache.MemberCache do @doc false defdelegate bulk_create(guild_id, members), to: @configured_cache - @doc """ - Return the QLC handle of the configured cache. - """ - @doc since: "0.8.0" - defdelegate query_handle(), to: @configured_cache - ## Supervisor callbacks # These set up the backing cache. @doc false diff --git a/lib/nostrum/cache/member_cache/ets.ex b/lib/nostrum/cache/member_cache/ets.ex index ec5b3d601..f4776be43 100644 --- a/lib/nostrum/cache/member_cache/ets.ex +++ b/lib/nostrum/cache/member_cache/ets.ex @@ -44,17 +44,17 @@ defmodule Nostrum.Cache.MemberCache.ETS do end @doc """ - Wrap QLC operations. + Wrap long-running queries operations. ## Safety {: .note} - Any QLC operations are surrounded by `:ets.safe_fixtable`. It is therefore + Any operations are surrounded by `:ets.safe_fixtable`. It is therefore recommended to finish your read quickly. """ - @doc since: "0.8.0" + @doc since: "0.10.0" @impl MemberCache - @spec wrap_qlc((-> qlc_result)) :: qlc_result when qlc_result: term() - def wrap_qlc(fun) do + @spec wrap_query((-> query_result)) :: query_result when query_result: term() + def wrap_query(fun) do :ets.safe_fixtable(@table_name, true) fun.() after @@ -76,6 +76,40 @@ defmodule Nostrum.Cache.MemberCache.ETS do end end + @impl MemberCache + @doc since: "0.10.0" + def by_user(user_id) do + ms = [{{{:"$1", user_id}, :"$2"}, [], [{{:"$1", :"$2"}}]}] + Stream.resource( + fn -> :ets.select(@table_name, ms, 100) + end, + fn items -> + case items do + {matches, cont} -> + {matches, :ets.select(cont)} + :"$end_of_table" -> {:halt, nil} end + end, + fn _cont -> :ok end + ) + end + + @impl MemberCache + @doc since: "0.10.0" + def by_guild(guild_id) do + ms = [{{{guild_id, :_}, :"$1"}, [], [:"$1"]}] + Stream.resource( + fn -> :ets.select(@table_name, ms, 100) + end, + fn items -> + case items do + {matches, cont} -> + {matches, :ets.select(cont)} + :"$end_of_table" -> {:halt, nil} end + end, + fn _cont -> :ok end + ) + end + @doc "Add the given member to the given guild in the cache." @impl MemberCache @spec create(Guild.id(), map()) :: Member.t() @@ -130,12 +164,4 @@ defmodule Nostrum.Cache.MemberCache.ETS do new_members = Enum.map(members, &{{guild_id, &1.user.id}, Util.cast(&1, {:struct, Member})}) true = :ets.insert(@table_name, new_members) end - - @impl MemberCache - @doc "Get a QLC query handle for the member cache." - @doc since: "0.7.0" - @spec query_handle :: :qlc.query_handle() - def query_handle do - :ets.table(@table_name) - end end diff --git a/lib/nostrum/cache/member_cache/mnesia.ex b/lib/nostrum/cache/member_cache/mnesia.ex index 49c280a62..a94e53c04 100644 --- a/lib/nostrum/cache/member_cache/mnesia.ex +++ b/lib/nostrum/cache/member_cache/mnesia.ex @@ -76,6 +76,38 @@ if Code.ensure_loaded?(:mnesia) do end) end + @impl MemberCache + @doc since: "0.10.0" + def by_user(user_id) do + ms = [{{:_, :_, :"$1", user_id, :"$2"}, [], [{{:"$1", :"$2"}}]}] + Stream.resource( + fn -> :mnesia.select(@table_name, ms, 100, :read) end, + fn items -> + case items do + {matches, cont} -> + {matches, :mnesia.select(cont)} + :"$end_of_table" -> {:halt, nil} end + end, + fn _cont -> :ok end + ) + end + + @impl MemberCache + @doc since: "0.10.0" + def by_guild(guild_id) do + ms = [{{:_, :_, guild_id, :_, :"$1"}, [], [:"$1"]}] + Stream.resource( + fn -> :mnesia.select(@table_name, ms, 100, :read) end, + fn items -> + case items do + {matches, cont} -> + {matches, :mnesia.select(cont)} + :"$end_of_table" -> {:halt, nil} end + end, + fn _cont -> :ok end + ) + end + @impl MemberCache @doc "Add the given member to the given guild in the cache." @spec create(Guild.id(), map()) :: Member.t() @@ -158,16 +190,9 @@ if Code.ensure_loaded?(:mnesia) do end @impl MemberCache - @doc "Get a QLC query handle for the member cache." - @spec query_handle :: :qlc.query_handle() - def query_handle do - :mnesia.table(@table_name) - end - - @impl MemberCache - @doc "Wrap QLC operations in a transaction." - @doc since: "0.8.0" - def wrap_qlc(fun) do + @doc "Wrap long-running queries in a transaction." + @doc since: "0.10.0" + def wrap_query(fun) do :mnesia.activity(:sync_transaction, fun) end end diff --git a/lib/nostrum/cache/member_cache/noop.ex b/lib/nostrum/cache/member_cache/noop.ex index adeda5842..381d5c4fb 100644 --- a/lib/nostrum/cache/member_cache/noop.ex +++ b/lib/nostrum/cache/member_cache/noop.ex @@ -22,6 +22,12 @@ defmodule Nostrum.Cache.MemberCache.NoOp do Supervisor.init([], strategy: :one_for_one) end + @impl MemberCache + def by_user(_user_id), do: [] + + @impl MemberCache + def by_guild(_guild_id), do: [] + @impl MemberCache def get(_guild_id, _user_id), do: {:error, :member_not_found} @@ -36,7 +42,4 @@ defmodule Nostrum.Cache.MemberCache.NoOp do @impl MemberCache def bulk_create(_guild_id, _members), do: true - - @impl MemberCache - def query_handle, do: :qlc.string_to_handle(~c"[].") end diff --git a/test/nostrum/cache/member_cache_meta_test.exs b/test/nostrum/cache/member_cache_meta_test.exs index 41eb7cc4d..303e0eb3f 100644 --- a/test/nostrum/cache/member_cache_meta_test.exs +++ b/test/nostrum/cache/member_cache_meta_test.exs @@ -59,10 +59,10 @@ defmodule Nostrum.Cache.MemberCacheMetaTest do [pid: start_supervised!(@cache)] end - if function_exported?(@cache, :wrap_qlc, 1) do - defdelegate wrap_qlc(fun), to: @cache + if function_exported?(@cache, :wrap_query, 1) do + defdelegate wrap_query(fun), to: @cache else - defp wrap_qlc(fun), do: fun.() + defp wrap_query(fun), do: fun.() end defp all_entries do