Skip to content

Commit 8a9d38e

Browse files
authored
Late Spring cleaning of Node lib indexing (#209)
* move node_module indexing logic into a separate namespace * refactor into smaller units of functionality * add unit tests * enhance export processing a little
1 parent 47d43be commit 8a9d38e

File tree

4 files changed

+274
-134
lines changed

4 files changed

+274
-134
lines changed

src/main/clojure/cljs/closure.clj

Lines changed: 1 addition & 133 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@
1414
[cljs.analyzer :as ana]
1515
[cljs.source-map :as sm]
1616
[cljs.env :as env]
17+
[cljs.foreign.node :refer [package-json-entries node-file-seq->libs-spec*]]
1718
[cljs.js-deps :as deps]
1819
[clojure.java.io :as io]
1920
[clojure.java.shell :as sh]
@@ -1958,35 +1959,6 @@
19581959
(apply str))
19591960
(.toSource closure-compiler ast-root)))))
19601961

1961-
(defn- package-json-entries
1962-
"Takes options and returns a sequence with the desired order of package.json
1963-
entries for the given :package-json-resolution mode. If no mode is provided,
1964-
defaults to :webpack (if no target is set) and :nodejs (if the target is
1965-
:nodejs)."
1966-
[opts]
1967-
{:pre [(or (= (:package-json-resolution opts) :webpack)
1968-
(= (:package-json-resolution opts) :nodejs)
1969-
(and (sequential? (:package-json-resolution opts))
1970-
(every? string? (:package-json-resolution opts)))
1971-
(not (contains? opts :package-json-resolution)))]}
1972-
(let [modes {:nodejs ["main"]
1973-
:webpack ["browser" "module" "main"]}]
1974-
(if-let [mode (:package-json-resolution opts)]
1975-
(if (sequential? mode) mode (get modes mode))
1976-
(case (:target opts)
1977-
:nodejs (:nodejs modes)
1978-
(:webpack modes)))))
1979-
1980-
(comment
1981-
(= (package-json-entries {}) ["browser" "module" "main"])
1982-
(= (package-json-entries {:package-json-resolution :nodejs}) ["main"])
1983-
(= (package-json-entries {:package-json-resolution :webpack}) ["browser" "module" "main"])
1984-
(= (package-json-entries {:package-json-resolution ["foo" "bar" "baz"]}) ["foo" "bar" "baz"])
1985-
(= (package-json-entries {:target :nodejs}) ["main"])
1986-
(= (package-json-entries {:target :nodejs :package-json-resolution :nodejs}) ["main"])
1987-
(= (package-json-entries {:target :nodejs :package-json-resolution :webpack}) ["browser" "module" "main"])
1988-
(= (package-json-entries {:target :nodejs :package-json-resolution ["foo" "bar"]}) ["foo" "bar"]))
1989-
19901962
(defn- sorting-dependency-options []
19911963
(try
19921964
(if (contains? (:flags (clojure.reflect/reflect DependencyOptions)) :abstract)
@@ -2770,110 +2742,6 @@
27702742
(get-in @env/*compiler* [::transitive-dep-set modules])))))))
27712743
(filterv identity))))
27722744

2773-
(defn- node-file-seq->libs-spec*
2774-
"Given a sequence of non-nested node_module paths where the extension ends in
2775-
`.js/.json`, return lib-spec maps for each path containing at least :file,
2776-
:module-type, and :provides."
2777-
[module-fseq opts]
2778-
(letfn [(package-json? [path]
2779-
(= "package.json" (.getName (io/file path))))
2780-
2781-
(top-level-package-json? [path]
2782-
(boolean (re-find #"node_modules[/\\](@[^/\\]+?[/\\])?[^/\\]+?[/\\]package\.json$" path)))
2783-
2784-
;; the path sans the package.json part
2785-
;; i.e. some_lib/package.json -> some_lib
2786-
(trim-package-json [s]
2787-
(if (string/ends-with? s "package.json")
2788-
(subs s 0 (- (count s) 12))
2789-
s))
2790-
2791-
(trim-relative [path]
2792-
(cond-> path
2793-
(string/starts-with? path "./")
2794-
(subs 2)))
2795-
2796-
(add-exports [pkg-jsons]
2797-
(reduce-kv
2798-
(fn [pkg-jsons path {:strs [exports] :as pkg-json}]
2799-
;; "exports" can just be a dupe of "main", i.e. a string - ignore
2800-
;; https://nodejs.org/api/packages.html#main-entry-point-export
2801-
(if (string? exports)
2802-
pkg-jsons
2803-
(reduce-kv
2804-
(fn [pkg-jsons export _]
2805-
;; NOTE: ignore "." exports for now
2806-
(if (= "." export)
2807-
pkg-jsons
2808-
(let [export-pkg-json
2809-
(io/file
2810-
(trim-package-json path)
2811-
(trim-relative export)
2812-
"package.json")]
2813-
(cond-> pkg-jsons
2814-
(.exists export-pkg-json)
2815-
(assoc
2816-
(.getAbsolutePath export-pkg-json)
2817-
(json/read-str (slurp export-pkg-json)))))))
2818-
pkg-jsons exports)))
2819-
pkg-jsons pkg-jsons))]
2820-
(let [
2821-
;; a map of all the *top-level* package.json paths and their exports
2822-
;; to the package.json contents as EDN
2823-
pkg-jsons (add-exports
2824-
(into {}
2825-
(comp
2826-
(map #(.getAbsolutePath %))
2827-
(filter top-level-package-json?)
2828-
(map (fn [path]
2829-
[path (json/read-str (slurp path))])))
2830-
module-fseq))]
2831-
(into []
2832-
(comp
2833-
(map #(.getAbsolutePath %))
2834-
(map (fn [path]
2835-
(merge
2836-
{:file path
2837-
:module-type :es6}
2838-
;; if the file is *not* a package.json, then compute what
2839-
;; namespaces it :provides to ClojureScript
2840-
(when-not (package-json? path)
2841-
(let [pkg-json-main (some
2842-
(fn [[pkg-json-path {:as pkg-json :strs [name]}]]
2843-
(let [entries (package-json-entries opts)
2844-
entry (first (keep (partial get pkg-json) entries))]
2845-
(when-not (nil? entry)
2846-
;; should be the only edge case in
2847-
;; the package.json main field - Antonio
2848-
(let [entry (trim-relative entry)
2849-
entry-path (-> pkg-json-path
2850-
(string/replace \\ \/)
2851-
trim-package-json
2852-
(str entry))]
2853-
;; find a package.json entry point that matches
2854-
;; the `path`
2855-
(some (fn [candidate]
2856-
(when (= candidate (string/replace path \\ \/))
2857-
name))
2858-
(cond-> [entry-path]
2859-
(not (or (string/ends-with? entry-path ".js")
2860-
(string/ends-with? entry-path ".json")))
2861-
(into [(str entry-path ".js") (str entry-path "/index.js") (str entry-path ".json")
2862-
(string/replace entry-path #"\.cjs$" ".js")])))))))
2863-
pkg-jsons)]
2864-
{:provides (let [module-rel-name (-> (subs path (.lastIndexOf path "node_modules"))
2865-
(string/replace \\ \/)
2866-
(string/replace #"node_modules[\\\/]" ""))
2867-
provides (cond-> [module-rel-name (string/replace module-rel-name #"\.js(on)?$" "")]
2868-
(some? pkg-json-main)
2869-
(conj pkg-json-main))
2870-
index-replaced (string/replace module-rel-name #"[\\\/]index\.js(on)?$" "")]
2871-
(cond-> provides
2872-
(and (boolean (re-find #"[\\\/]index\.js(on)?$" module-rel-name))
2873-
(not (some #{index-replaced} provides)))
2874-
(conj index-replaced)))}))))))
2875-
module-fseq))))
2876-
28772745
(def node-file-seq->libs-spec (memoize node-file-seq->libs-spec*))
28782746

28792747
(defn index-node-modules-dir
Lines changed: 185 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,185 @@
1+
(ns cljs.foreign.node
2+
(:require [cljs.vendor.clojure.data.json :as json]
3+
[clojure.java.io :as io]
4+
[clojure.string :as string]))
5+
6+
(defn package-json-entries
7+
"Takes options and returns a sequence with the desired order of package.json
8+
entries for the given :package-json-resolution mode. If no mode is provided,
9+
defaults to :webpack (if no target is set) and :nodejs (if the target is
10+
:nodejs)."
11+
[opts]
12+
{:pre [(or (= (:package-json-resolution opts) :webpack)
13+
(= (:package-json-resolution opts) :nodejs)
14+
(and (sequential? (:package-json-resolution opts))
15+
(every? string? (:package-json-resolution opts)))
16+
(not (contains? opts :package-json-resolution)))]}
17+
(let [modes {:nodejs ["main"]
18+
:webpack ["browser" "module" "main"]}]
19+
(if-let [mode (:package-json-resolution opts)]
20+
(if (sequential? mode) mode (get modes mode))
21+
(case (:target opts)
22+
:nodejs (:nodejs modes)
23+
(:webpack modes)))))
24+
25+
(comment
26+
(= (package-json-entries {}) ["browser" "module" "main"])
27+
(= (package-json-entries {:package-json-resolution :nodejs}) ["main"])
28+
(= (package-json-entries {:package-json-resolution :webpack}) ["browser" "module" "main"])
29+
(= (package-json-entries {:package-json-resolution ["foo" "bar" "baz"]}) ["foo" "bar" "baz"])
30+
(= (package-json-entries {:target :nodejs}) ["main"])
31+
(= (package-json-entries {:target :nodejs :package-json-resolution :nodejs}) ["main"])
32+
(= (package-json-entries {:target :nodejs :package-json-resolution :webpack}) ["browser" "module" "main"])
33+
(= (package-json-entries {:target :nodejs :package-json-resolution ["foo" "bar"]}) ["foo" "bar"]))
34+
35+
(defn- package-json? [path]
36+
(= "package.json" (.getName (io/file path))))
37+
38+
(defn- top-level-package-json? [path]
39+
(boolean (re-find #"node_modules[/\\](@[^/\\]+?[/\\])?[^/\\]+?[/\\]package\.json$" path)))
40+
41+
;; the path sans the package.json part
42+
;; i.e. some_lib/package.json -> some_lib
43+
(defn- trim-package-json [s]
44+
(if (string/ends-with? s "package.json")
45+
(subs s 0 (- (count s) 12))
46+
s))
47+
48+
(defn- trim-relative [path]
49+
(cond-> path
50+
(string/starts-with? path "./")
51+
(subs 2)))
52+
53+
(defn- ->export-pkg-json [path export]
54+
(io/file
55+
(trim-package-json path)
56+
(trim-relative export)
57+
"package.json"))
58+
59+
(defn- export-subpaths
60+
"Examine the export subpaths to compute subpackages"
61+
[pkg-jsons export-subpath export path pkg-name]
62+
;; NOTE: ignore "." exports for now
63+
(if (= "." export-subpath)
64+
pkg-jsons
65+
;; technically the following logic is a bit brittle since `exports` is
66+
;; supposed to be used to hide the package structure.
67+
;; instead, here we assume the export subpath does match the library structure
68+
;; on disk, if we find a package.json we add it to pkg-jsons map
69+
;; and we synthesize "name" key based on subpath
70+
(let [export-pkg-json (->export-pkg-json path export-subpath)]
71+
;; note this will ignore export wildcards etc.
72+
(cond-> pkg-jsons
73+
(.exists export-pkg-json)
74+
(-> (assoc
75+
(.getAbsolutePath export-pkg-json)
76+
(merge
77+
(json/read-str (slurp export-pkg-json))
78+
;; add the name field so that path->main-name works later
79+
(when (and (map? export)
80+
(contains? export "require"))
81+
{"name" (str pkg-name (string/replace export-subpath "./" "/"))}))))))))
82+
83+
(defn- add-exports
84+
"Given a list of pkg-jsons examine them for the `exports` field. `exports`
85+
is now the preferred way to declare an entrypoint to a Node.js library. However,
86+
for backwards compatibility it is often combined with `main`.
87+
88+
`export` can also be a JS object - if so, it can define subpaths. `.` points
89+
to main and other subpaths can be defined relative to that.
90+
91+
See https://nodejs.org/api/packages.html#main-entry-point-export for more
92+
detailed information."
93+
[pkg-jsons opts]
94+
(reduce-kv
95+
(fn [pkg-jsons path {:strs [exports] :as pkg-json}]
96+
(if (string? exports)
97+
pkg-jsons
98+
;; map case
99+
(reduce-kv
100+
(fn [pkg-jsons export-subpath export]
101+
(export-subpaths pkg-jsons
102+
export-subpath export path (get pkg-json "name")))
103+
pkg-jsons exports)))
104+
pkg-jsons pkg-jsons))
105+
106+
(defn path->main-name
107+
"Determine whether a path is a main entrypoint in the provided package.json.
108+
If so return the name entry provided in the package.json file."
109+
[path [pkg-json-path {:as pkg-json :strs [name]}] opts]
110+
(let [entries (package-json-entries opts)
111+
entry (first (keep (partial get pkg-json) entries))]
112+
(when-not (nil? entry)
113+
;; should be the only edge case in
114+
;; the package.json main field - Antonio
115+
(let [entry (trim-relative entry)
116+
entry-path (-> pkg-json-path (string/replace \\ \/)
117+
trim-package-json (str entry))]
118+
;; find a package.json entry point that matches
119+
;; the `path`
120+
(some (fn [candidate]
121+
(when (= candidate (string/replace path \\ \/)) name))
122+
(cond-> [entry-path]
123+
;; if we have an entry point that doesn't end in .js or .json
124+
;; try to match some alternatives
125+
(not (or (string/ends-with? entry-path ".js")
126+
(string/ends-with? entry-path ".json")))
127+
(into [(str entry-path ".js") (str entry-path "/index.js") (str entry-path ".json")
128+
(string/replace entry-path #"\.cjs$" ".js")])))))))
129+
130+
(defn- path->rel-name [path]
131+
(-> (subs path (.lastIndexOf path "node_modules"))
132+
(string/replace \\ \/)
133+
(string/replace #"node_modules[\\\/]" "")))
134+
135+
(defn path->provides
136+
"For a given path in node_modules, determine what namespaces that file would
137+
provide to ClojureScript. Note it is assumed that we *already* processed all
138+
package.json files and they are present via pkg-jsons parameter as we need them
139+
to figure out the provides."
140+
[path pkg-jsons opts]
141+
(merge
142+
{:file path
143+
:module-type :es6}
144+
;; if the file is *not* a package.json, then compute what
145+
;; namespaces it :provides to ClojureScript
146+
(when-not (package-json? path)
147+
;; given some path search the package.json to determine whether it is a
148+
;; main entry point or not
149+
(let [pkg-json-main (some #(path->main-name path % opts) pkg-jsons)]
150+
{:provides (let [module-rel-name (path->rel-name path)
151+
provides (cond-> [module-rel-name (string/replace module-rel-name #"\.js(on)?$" "")]
152+
(some? pkg-json-main) (conj pkg-json-main))
153+
index-replaced (string/replace module-rel-name #"[\\\/]index\.js(on)?$" "")]
154+
(cond-> provides
155+
(and (boolean (re-find #"[\\\/]index\.js(on)?$" module-rel-name))
156+
(not (some #{index-replaced} provides)))
157+
(conj index-replaced)))}))))
158+
159+
(defn get-pkg-jsons
160+
"Given all a seq of files in node_modules return a map of all package.json
161+
files indexed by path. Includes any `export` package.json files as well"
162+
([module-fseq]
163+
(get-pkg-jsons module-fseq nil))
164+
([module-fseq opts]
165+
(add-exports
166+
(into {}
167+
(comp (map #(.getAbsolutePath %))
168+
(filter top-level-package-json?)
169+
(map (fn [path] [path (json/read-str (slurp path))])))
170+
module-fseq) opts)))
171+
172+
(defn node-file-seq->libs-spec*
173+
"Given a sequence of non-nested node_module paths where the extension ends in
174+
`.js/.json`, return lib-spec maps for each path containing at least :file,
175+
:module-type, and :provides."
176+
[module-fseq opts]
177+
(let [;; a map of all the *top-level* package.json paths and their exports
178+
;; to the package.json contents as EDN
179+
pkg-jsons (get-pkg-jsons module-fseq opts)]
180+
(into []
181+
(comp
182+
(map #(.getAbsolutePath %))
183+
;; for each file, figure out what it will provide to ClojureScript
184+
(map #(path->provides % pkg-jsons opts)))
185+
module-fseq)))

src/main/clojure/cljs/util.cljc

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -403,7 +403,8 @@
403403
(filter (fn [^File f]
404404
(let [path (.getPath f)]
405405
(or (.endsWith path ".json")
406-
(.endsWith path ".js"))))
406+
(.endsWith path ".js")
407+
(.endsWith path ".cjs"))))
407408
fseq)))
408409

409410
(defn node-path-modules [opts]

0 commit comments

Comments
 (0)