Set proper cache dir for posix build

.github/alpine/liquidsoap.pre-install

@@ -3,5 +3,9 @@
 addgroup -S liquidsoap 2> /dev/null
 adduser -S -D -h /var/liquidsoap -s /sbin/nologin -G liquidsoap -g liquidsoap liquidsoap 2> /dev/null
 addgroup liquidsoap audio 2> /dev/null
+mkdir -p /var/log/liquidsoap
+mkdir -p /var/cache/liquidspap
+chown liquidsoap:liquidsoap /var/log/liquidsoap
+chown liquidsoap:liquidsoap /var/cache/liquidsoap
 
 exit 0

.github/debian/dirs

@@ -2,3 +2,4 @@ usr/bin
 etc/init.d
 etc/bash_completion.d
 var/log/liquidsoap
+var/cache/liquidsoap

doc/content/language.md

@@ -1434,34 +1434,82 @@ During the first execution, the script is parsed, type checked and evaluated. On
 Here's a log without caching on a M3 macbook pro:
 
 ```
-2024/05/25 18:38:00 [startup:3] Cache retrieval: 0.01s
-2024/05/25 18:38:00 [startup:3] Typechecking: 2.58s
-2024/05/25 18:38:00 [startup:3] Evaluation: 0.01s
+2024/07/03 14:31:41 [startup:3] main script hash computation: 0.03s
+2024/07/03 14:31:41 [startup:3] main script cache retrieval: 0.03s
+2024/07/03 14:31:41 [startup:3] stdlib hash computation: 0.03s
+2024/07/03 14:31:41 [startup:3] stdlib cache retrieval: 0.03s
+2024/07/03 14:31:41 [startup:3] Typechecking stdlib: 3.37s
+2024/07/03 14:31:41 [startup:3] Typechecking main script: 0.00s
 ```
 
 And the same log after caching:
 
 ```
-2024/05/25 18:38:27 [startup:3] Loading script from cache!
-2024/05/25 18:38:27 [startup:3] Cache retrieval: 0.03s
-2024/05/25 18:38:27 [startup:3] Evaluation: 0.02s
+2024/07/03 14:32:59 [startup:3] main script hash computation: 0.02s
+2024/07/03 14:32:59 [startup:3] Loading main script from cache!
+2024/07/03 14:32:59 [startup:3] main script cache retrieval: 0.05s
 ```
 
 Scripts can be cached ahead of time without executing them, for instance while compiling a docker image, using `--cache-only`. Caching can also be disabled using `--no-cache`.
 
-On windows, the default cache directory is located in the same directory as the binary. On unix systems, it is located at: `$HOME/.cache/liquidsoap`
+Caching happens at two different time:
 
-For obvious reasons, cache parameters have to be set before parsing and executing scripts so:
+- First the standard library is cached
+- Then the script itself is cached
 
-- cache directory can be changed using the `LIQ_CACHE_DIR` environments
-- cache can be disabled by setting `LIQ_CACHE` to anything else than `"true"`
+Caching the standard library makes it possible to run the type-checker faster on new scripts. Here's an example of a log from running a new script with
+a cached standard library:
 
-At runtime, `liquidsoap.config()` returns the cache directory.
+```
+2024/07/03 14:33:27 [startup:3] main script hash computation: 0.02s
+2024/07/03 14:33:27 [startup:3] main script cache retrieval: 0.02s
+2024/07/03 14:33:27 [startup:3] stdlib hash computation: 0.03s
+2024/07/03 14:33:27 [startup:3] Loading stdlib from cache!
+2024/07/03 14:33:27 [startup:3] stdlib cache retrieval: 0.10s
+2024/07/03 14:33:27 [startup:3] Typechecking main script: 0.00s
+```
+
+Caching can be disabled by setting `LIQ_CACHE` to anything else than `"true"`.
+
+### Cache locations
+
+Cache files can accumulate and also take up disk space so it is important to know where they are located!
+
+There are two type of cache locations:
+
+- System cache for cached files that should be shared with all liquidsoap scripts. This is where the standard library cache is located. This location is a system-wide path on unix system such as `/var/cache/liquidsoap`.
+- User cache for cached files that are specific to the user running liquidsoap scripts. On unix systems, this location is at `$HOME/.cache/liquidsoap`.
+
+On windows, the default cache directory for both type of cache locations is in the same directory as the binary.
+
+At runtime, `liquidsoap.cache(mode=<mode>)` returns the cache directory. `mode` should be one of: `"user"` or `"system"`.
+
+### Cache maintenance
+
+There is a cache maintenance routine which deletes unused cache files after `10` days and keeps the cache to a maximum of `200` files.
+
+You can run the cache maintenance routing by calling `liquidsoap.cache.maintenance(mode=<mode>)` manually. Here, too, `mode` should be one of: `"user"` or `"system"`.
+
+### Cache security
+
+Please be aware that the cache does _not_ encrypt its values. As such, user cache files should be considered sensitive as they may contain password and other runtime secrets
+that are available through your scripts. We recommend to:
+
+- Use environment variables as much as possible when passing secrets
+- Secure your user script and cache files.
+
+The default creation permissions for user cache files is: `0o600` so only the user creating them should be able to read them. You should make sure that your script permissions are also similarly restricted.
 
-There is a cache maintenance routine which deletes unused cache files after `10` days and keeps the cache to a maximum of `200` files. This can be configured
-via `settings.cache.max_day` and `settings.cache.max_files`.
+### Cache environment variables
 
-However, also for obvious reasons, these values can only be changed _after_ executing the script so setting a values in your script will not affect the initial maintenance done
-during the script's loading.
+The following environment variables control the cache behavior:
 
-Thus, if you need to change the defaults and run the cache maintenance, you can configure the values in your script and run `liquidsoap.cache.maintenance()` manually.
+- `LIQ_CACHE`: disable the cache when set to anything else than `1` or `true`
+- `LIQ_CACHE_SYSTEM_DIR`: set the cache system directory
+- `LIQ_CACHE_SYSTEM_DIR_PERMS`: set the permission used when creating cache system directory (and its parents when needed). Default: `0o755`
+- `LIQ_CACHE_SYSTEM_FILE_PERMS`: set the permissions used when creating a system cache file. Default: `0o644`
+- `LIQ_CACHE_USER_DIR`: set the cache user directory
+- `LIQ_CACHE_USER_DIR_PERMS`: set the permission used when creating cache user directory (and its parents when needed). Default: `0o700`.
+- `LIQ_CACHE_USER_FILE_PERMS`: set the permissions used when creating a user cache file. Default: `0o600`
+- `LIQ_CACHE_MAX_DAYS`: set the maximum days a cache file can be stored before it is eligible to be deleted during the next cache maintenance pass.
+- `LIQ_CACHE_MAX_FILES`: set the maximum number of files in each cache directory. Older files are removed first.

dune-project

@@ -159,7 +159,7 @@
     (sedlex (>= 3.2))
     (menhir (>= 20180703))
   )
-  (sites (share libs) (share bin) (lib_root lib_root))
+  (sites (share libs) (share bin) (share cache) (lib_root lib_root))
   (synopsis "Liquidsoap language library"))
 
 (package

src/config/liquidsoap_paths.default.ml

@@ -23,3 +23,10 @@ let bin_dir () = get_site Sites.bin
 let bin_dir_descr = "(set by dune-site)"
 let camomile_dir () = Filename.dirname CamomileLib.Config.Default.datadir
 let camomile_dir_descr = "(set by dune-site)"
+let user_cache_override () = None
+let user_cache_override_descr = "$HOME/.cache/liquidsoap"
+
+let system_cache_override () =
+  match Sites.cache with [] -> None | d :: _ -> Some d
+
+let system_cache_override_descr = "(set by dune-site)"

src/config/liquidsoap_paths.posix.ml

@@ -11,3 +11,7 @@ let bin_dir () = "/usr/share/liquidsoap/bin"
 let bin_dir_descr = bin_dir ()
 let camomile_dir () = "/usr/share/liquidsoap/camomile"
 let camomile_dir_descr = camomile_dir ()
+let user_cache_override () = None
+let user_cache_override_descr = "$HOME/.cache/liquidsoap"
+let system_cache_override () = Some "/var/cache/liquidsoap"
+let system_cache_override_descr = "/var/cache/liquidsoap"

src/config/liquidsoap_paths.standalone.ml

@@ -12,3 +12,15 @@ let bin_dir () = path "bin"
 let bin_dir_descr = "./bin"
 let camomile_dir () = path "camomile"
 let camomile_dir_descr = "./camomile"
+
+let user_cache_override () =
+  let dir = Filename.dirname Sys.executable_name in
+  let cwd = Sys.getcwd () in
+  Sys.chdir dir;
+  let dir = Sys.getcwd () in
+  Sys.chdir cwd;
+  Some (Filename.concat dir ".cache")
+
+let user_cache_override_descr = "./cache"
+let system_cache_override () = Some "./cache"
+let system_cache_override_descr = "./cache"

src/core/configure.ml

@@ -6,6 +6,12 @@ let git_snapshot = git_sha <> None
 let requests_max_id = 50
 let requests_table_size = 50
 
+let () =
+  Liquidsoap_lang.Cache.user_dir_override :=
+    Liquidsoap_paths.user_cache_override;
+  Liquidsoap_lang.Cache.system_dir_override :=
+    Liquidsoap_paths.system_cache_override
+
 (** General configuration *)
 let conf = Dtools.Conf.void "Liquidsoap configuration"
 

src/core/hooks_implementations.ml

@@ -131,30 +131,38 @@ let register () =
   Hooks.source_methods_t :=
     fun () -> Lang_source.source_t ~methods:true (Lang.univ_t ())
 
-let conf_cache =
-  Dtools.Conf.void ~p:(Configure.conf#plug "cache") "Cache configuration"
+let cache_max_days =
+  try int_of_string (Sys.getenv "LIQ_CACHE_MAX_DAYS") with _ -> 10
 
-let conf_cache_max_days =
-  Dtools.Conf.int
-    ~p:(conf_cache#plug "max_days")
-    ~d:10
-    "Delete cache file that have not been touched after this number of days."
+let cache_max_files =
+  try int_of_string (Sys.getenv "LIQ_CACHE_MAX_FILES") with _ -> 20
 
-let conf_cache_max_files =
-  Dtools.Conf.int
-    ~p:(conf_cache#plug "max_files")
-    ~d:200 "Keep at most that number of cache files. Delete folder files first."
+let () =
+  (try
+     Liquidsoap_lang.Cache.system_dir_perms :=
+       int_of_string (Sys.getenv "LIQ_CACHE_SYSTEM_DIR_PERMS")
+   with _ -> ());
+  (try
+     Liquidsoap_lang.Cache.system_file_perms :=
+       int_of_string (Sys.getenv "LIQ_CACHE_SYSTEM_FILE_PERMS")
+   with _ -> ());
+  (try
+     Liquidsoap_lang.Cache.user_dir_perms :=
+       int_of_string (Sys.getenv "LIQ_CACHE_USER_DIR_PERMS")
+   with _ -> ());
+  try
+    Liquidsoap_lang.Cache.user_file_perms :=
+      int_of_string (Sys.getenv "LIQ_CACHE_USER_FILE_PERMS")
+  with _ -> ()
 
 module Term_cache = Liquidsoap_lang.Term_cache
 
 let cache_log = Log.make ["cache"]
 
-let cache_maintenance () =
-  let max_timestamp =
-    Unix.time () -. (float conf_cache_max_days#get *. 86400.)
-  in
+let cache_maintenance dirtype =
+  let max_timestamp = Unix.time () -. (float cache_max_days *. 86400.) in
   try
-    match Cache.dir () with
+    match Cache.dir dirtype with
       | Some dir when Sys.file_exists dir && Sys.is_directory dir ->
           let files =
             Array.fold_left
@@ -172,9 +180,8 @@ let cache_maintenance () =
               [] (Sys.readdir dir)
           in
           let len = List.length files in
-          let max_files = conf_cache_max_files#get in
-          if max_files < len then (
-            let len = len - max_files in
+          if cache_max_files < len then (
+            let len = len - cache_max_files in
             cache_log#info "Too many cached files! Deleting %d oldest ones.."
               len;
             let files =

src/config/liquidsoap_paths.mli → src/core/liquidsoap_paths.mli

@@ -11,3 +11,7 @@ val bin_dir : unit -> string
 val bin_dir_descr : string
 val camomile_dir : unit -> string
 val camomile_dir_descr : string
+val user_cache_override : unit -> string option
+val user_cache_override_descr : string
+val system_cache_override : unit -> string option
+val system_cache_override_descr : string

src/core/operators/ladspa_op.ml

@@ -433,7 +433,7 @@ let register_plugin cache pname =
 
 let register_plugins () =
   let cache =
-    (Cache.Table.load ~name:"LADSPA plugins" "ladspa-plugins"
+    (Cache.Table.load ~dirtype:`System ~name:"LADSPA plugins" "ladspa-plugins"
       : plugin Cache.Table.t)
   in
   let add plugins_dir =
@@ -451,7 +451,7 @@ let register_plugins () =
         (Unix.error_message e)
   in
   List.iter add ladspa_dirs;
-  Cache.Table.store cache
+  Cache.Table.store ~dirtype:`System cache
 
 let () =
   Lifecycle.on_load ~name:"ladspa plugin registration" (fun () ->

src/core/operators/lilv_op.ml

@@ -410,13 +410,13 @@ let register_plugin cache plugin =
 
 let register_plugins () =
   let cache =
-    (Cache.Table.load ~name:"lilv plugins" "lilv-plugins"
+    (Cache.Table.load ~dirtype:`System ~name:"lilv plugins" "lilv-plugins"
       : plugin Cache.Table.t)
   in
   let world = World.create () in
   World.load_all world;
   Plugins.iter (register_plugin cache) (World.plugins world);
-  Cache.Table.store cache
+  Cache.Table.store ~dirtype:`System cache
 
 let () =
   Lifecycle.on_load ~name:"lilv plugin registration" (fun () ->

src/lang/builtins_lang.ml

@@ -136,15 +136,55 @@ let liquidsoap = Modules.liquidsoap
 
 let liquidsoap_cache =
   Lang.add_builtin ~category:`Configuration ~descr:"Liquidsoap cache directory."
-    ~base:liquidsoap "cache" [] (Lang.nullable_t Lang.string_t) (fun _ ->
-      match Cache.dir () with None -> Lang.null | Some dir -> Lang.string dir)
+    ~base:liquidsoap "cache"
+    [
+      ( "mode",
+        Lang.string_t,
+        None,
+        Some "Cache mode, one of: \"user\" or \"system\"" );
+    ]
+    (Lang.nullable_t Lang.string_t)
+    (fun p ->
+      let mode = List.assoc "mode" p in
+      let dirtype =
+        match Lang.to_string mode with
+          | "system" -> `System
+          | "user" -> `User
+          | _ ->
+              raise
+                (Error.Invalid_value
+                   ( mode,
+                     "Invalid mode. Should be one of: \"user\" or \"system\"" ))
+      in
+      match Cache.dir dirtype with
+        | None -> Lang.null
+        | Some dir -> Lang.string dir)
 
 let _ =
   Lang.add_builtin ~category:`Configuration
     ~descr:"Execute cache maintenance routine." ~base:liquidsoap_cache
-    "maintenance" [] Lang.unit_t (fun _ ->
+    "maintenance"
+    [
+      ( "mode",
+        Lang.string_t,
+        None,
+        Some "Cache mode, one of: \"user\" or \"system\"" );
+    ]
+    Lang.unit_t
+    (fun p ->
+      let mode = List.assoc "mode" p in
+      let dirtype =
+        match Lang.to_string mode with
+          | "system" -> `System
+          | "user" -> `User
+          | _ ->
+              raise
+                (Error.Invalid_value
+                   ( mode,
+                     "Invalid mode. Should be one of: \"user\" or \"system\"" ))
+      in
       let fn = !Hooks.cache_maintenance in
-      fn ();
+      fn dirtype;
       Lang.unit)
 
 let liquidsoap_version =