cmj cache

Author: cknitt

.github/workflows/ci.yml

@@ -123,8 +123,10 @@ jobs:
  if: runner.os == 'Linux'
  uses: actions/upload-artifact@v3
  with:
- name: lib-cmi-cache
- path: lib/cmi_cache.bin
+ name: cmij-cache
+ path: |
+ lib/cmi_cache.bin
+ lib/cmj_cache.bin
 
  - name: Check for changes in lib folder
  run: git diff --exit-code lib/js lib/es6

jscomp/cmij/cmij.ml

@@ -0,0 +1,8 @@
+type t = { module_names : string array; module_data : bytes array }
+type cmi_data = Cmi_format.cmi_infos
+type cmj_data = { values : Js_cmj_format.keyed_cmj_value array; pure : bool }
+
+let marshal_cmi_data (cmi_data : cmi_data) = Marshal.to_bytes cmi_data []
+let marshal_cmj_data (cmj_data : cmj_data) = Marshal.to_bytes cmj_data []
+let unmarshal_cmi_data bytes : cmi_data = Marshal.from_bytes bytes 0
+let unmarshal_cmj_data bytes : cmj_data = Marshal.from_bytes bytes 0

jscomp/cmij/cmij.mli

@@ -0,0 +1,8 @@
+type t = { module_names : string array; module_data : bytes array }
+type cmi_data = Cmi_format.cmi_infos
+type cmj_data = { values : Js_cmj_format.keyed_cmj_value array; pure : bool }
+
+val marshal_cmi_data : cmi_data -> bytes
+val marshal_cmj_data : cmj_data -> bytes
+val unmarshal_cmi_data : bytes -> cmi_data
+val unmarshal_cmj_data : bytes -> cmj_data

jscomp/cmij/cmij_cache.ml

@@ -30,32 +30,17 @@ let get_files ext dir =
  if Ext_string.ends_with x ext then Some (Filename.concat dir x) else None)
  |> Array.to_list
 
-(** the cache should be readable and also update *)
-let check_digest output_file digest : bool =
- if Sys.file_exists output_file then
- match
- Ext_list.filter
- (String.split_on_char ' ' (Ext_io.load_file output_file))
- (fun x -> x <> "")
- with
- | _head :: old_digest :: _tail -> Digest.equal digest old_digest
- | _ -> false
- else false
-
-let ( +> ) = Ext_buffer.add_string
-
 let from_cmj ~mode (files : string list) (output_file : string) : unit =
  let files =
  let cmp = Ext_filename.module_name in
- List.sort
- (fun filea fileb -> Ext_string_array.cmp (cmp filea) (cmp fileb))
- files
+ files
+ |> List.sort (fun filea fileb ->
+ Ext_string_array.cmp (cmp filea) (cmp fileb))
+ |> Array.of_list
  in
- let buf = Ext_buffer.create 10000 in
-
- let abs =
- Ext_list.map files (fun file ->
- let module_name = Ext_filename.module_name file in
+ let module_names = Ext_array.map files Ext_filename.module_name in
+ let module_data =
+ Ext_array.map files (fun file ->
  let content : Js_cmj_format.t = Js_cmj_format.from_file file in
  let () =
  match mode with
@@ -72,29 +57,13 @@ let from_cmj ~mode (files : string list) (output_file : string) : unit =
  | Playground _ -> ()
  in
  (* prerr_endline (Ext_obj.dump content.package_spec); *)
- let c = Marshal.to_string (content.values, content.pure) [] in
- ( Printf.sprintf {|%S (* %d *)|} module_name (String.length c),
- Printf.sprintf {|(* %s *)%S|} module_name c ))
+ let { Js_cmj_format.values; pure } = content in
+ Cmij_cache.marshal_cmj_data { values; pure })
  in
- buf
- +> Printf.sprintf
- {|let module_names : string array = Obj.magic (
-%s
-)
-let module_data : string array = Obj.magic (
-%s
-)
-|}
- (String.concat ",\n" (Ext_list.map abs fst))
- (String.concat ",\n" (Ext_list.map abs snd));
- buf +> "\n";
- let digest = Digest.to_hex (Ext_buffer.digest buf) in
- let same = check_digest output_file digest in
- if not same then
- let v = open_out_bin output_file in
- Ext_pervasives.finally v ~clean:close_out (fun f ->
- output_string f ("(* " ^ digest ^ " *) \n");
- Ext_buffer.output_buffer f buf)
+ let cmj_cache = { Cmij_cache.module_names; module_data } in
+ let v = open_out_bin output_file in
+ Ext_pervasives.finally v ~clean:close_out (fun f ->
+ Marshal.to_channel f cmj_cache [])
 
 let from_cmi files output_file =
  let files =
@@ -110,21 +79,19 @@ let from_cmi files output_file =
  let module_name = Ext_filename.module_name file in
  let cmi = Cmi_format.read_cmi file in
  assert (cmi.cmi_name = module_name);
- let content = Marshal.to_bytes cmi [] in
- content)
+ Cmij_cache.marshal_cmi_data cmi)
  in
- let cmi_cache = { Cmij.module_names; module_data } in
+ let cmi_cache = { Cmij_cache.module_names; module_data } in
  let v = open_out_bin output_file in
  Ext_pervasives.finally v ~clean:close_out (fun f ->
  Marshal.to_channel f cmi_cache [])
 
 let stdlib = "stdlib-406"
 let ( // ) = Filename.concat
 let ( |~ ) = Ext_string.contain_substring
-
-let cmi_target_file =
- Filename.dirname Sys.argv.(0) // ".." // ".." // "lib" // "cmi_cache.bin"
-
+let lib_dir = Filename.dirname Sys.argv.(0) // ".." // ".." // "lib"
+let cmi_target_file = lib_dir // "cmi_cache.bin"
+let cmj_target_file = lib_dir // "cmj_cache.bin"
 let release_cmi = Array.exists (( = ) "-release") Sys.argv
 
 let () =
@@ -158,9 +125,7 @@ let () =
  (Filename.dirname Sys.argv.(0) // ".." // "others")
  @ third_party_cmj_files
  in
- from_cmj ~mode cmj_files
- (Filename.dirname Sys.argv.(0)
- // ".." // "core" // "builtin_cmj_datasets.ml");
+ from_cmj ~mode cmj_files cmj_target_file;
  let third_party_cmi_files =
  match mode with
  | Native -> []

jscomp/cmij/cmij_cache.mli

@@ -25,15 +25,13 @@
 #ifdef RELEASE 
 (*true *)
 
-type cmi_cache = {module_names : string array; module_data : bytes array}
-
 let cmi_cache_path =
  let ( // ) = Filename.concat in
  Config.standard_library // Filename.parent_dir_name // "cmi_cache.bin"
 
 let load_cmi_cache () =
  let channel = open_in_bin cmi_cache_path in
- let cache: cmi_cache = Marshal.from_channel channel in
+ let cache: Cmij_cache.t = Marshal.from_channel channel in
  close_in channel;
  cache
 
@@ -45,13 +43,12 @@ let load_cmi ~unit_name : Env.Persistent_signature.t option =
  | None ->
  if !Js_config.no_stdlib then None
  else
- let {module_names; module_data} = Lazy.force cmi_cache in
+ let {Cmij_cache.module_names; module_data} = Lazy.force cmi_cache in
  match Ext_string_array.find_sorted module_names unit_name with
- | Some cmi ->
+ | Some index ->
  if Js_config.get_diagnose () then
  Format.fprintf Format.err_formatter ">Cmi: %s@." unit_name;
- let cmi : Cmi_format.cmi_infos = 
- Marshal.from_bytes module_data.(cmi) 0 in 
+ let cmi = Cmij_cache.unmarshal_cmi_data module_data.(index) in
  if Js_config.get_diagnose () then
  Format.fprintf Format.err_formatter "<Cmi: %s@." unit_name;
  Some {filename = Sys.executable_name ; 

jscomp/cmij/cmij_main.ml

@@ -11,3 +11,5 @@
 ; There exist cyclic dependencies between core and frontend, therefore include frontend files here:
 
 (copy_files# ../frontend/*.{ml,mli})
+
+(copy_files# ../cmij/cmij_cache.{ml,mli})