@@ -18,6 +18,51 @@ if (!dbFile || !hugoOutput) {
1818 process . exit ( 1 ) ;
1919}
2020
21+ /**
22+ * @param {string } u a spec URL like "specs.ipfs.tech/http-gateways/path-gateway/#if-none-match-request-header"
23+ * @returns the spec's parent, or "null" if it's a top-level spec
24+ */
25+ const computeParent = ( u ) => {
26+ const url = new URL ( u ) ;
27+ const segments = url . pathname . split ( '/' ) . filter ( Boolean ) ;
28+
29+ // if there's a hash, consider it as a segment
30+ if ( url . hash ) segments . push ( url . hash . substring ( 1 ) ) ;
31+
32+ if ( segments . length <= 1 ) {
33+ return "null" ;
34+ }
35+
36+ const parent = segments . slice ( 0 , - 1 ) . join ( '/' ) ;
37+ return `${ url . protocol } ${ url . host } ${ parent }
38+ } ;
39+
40+ /**
41+ * @param {string } u a spec URL like "specs.ipfs.tech/http-gateways/path-gateway/#if-none-match-request-header"
42+ * @returns the spec's name, or the hash if it's a top-level spec and whether it was found in a hash
43+ */
44+ const computeName = ( u ) => {
45+ const url = new URL ( u ) ;
46+
47+ if ( url . hash ) {
48+ return {
49+ isHashed : true ,
50+ name : url . hash . substring ( 1 ) ,
51+ } ;
52+ }
53+
54+ const segments = url . pathname . split ( '/' ) . filter ( Boolean ) ;
55+
56+ if ( segments . length === 0 ) {
57+ throw new Error ( `Invalid spec URL: ${ u } ) ;
58+ }
59+
60+ return {
61+ isHashed : false ,
62+ name : segments [ segments . length - 1 ] ,
63+ } ;
64+ } ;
65+
2166const main = async ( ) => {
2267 let db = new sqlite3 . Database ( dbFile , ( err ) => {
2368 if ( err ) {
@@ -58,6 +103,8 @@ const main = async () => {
58103 ` ;
59104 const testsRows = await all ( testsQuery ) ;
60105 const groups = { } ;
106+ const flatTestGroups = { } ; // used for specs generation.
107+
61108 for ( const row of testsRows ) {
62109 const { versions, full_name, name, parent_test_full_name } = row ;
63110 const slug = slugify ( full_name ) ;
@@ -66,10 +113,115 @@ const main = async () => {
66113 groups [ parent_test_full_name ] = { } ;
67114 }
68115
69- groups [ parent_test_full_name ] [ full_name ] = { versions : versions ?. split ( ',' ) || [ ] , name, full_name, slug } ;
116+ const g = { versions : versions ?. split ( ',' ) || [ ] , name, full_name, slug } ;
117+
118+ groups [ parent_test_full_name ] [ full_name ] = g ;
119+ flatTestGroups [ full_name ] = g ;
70120 }
71121 outputJSON ( "data/testgroups.json" , groups ) ;
72122
123+ // Query to fetch all test specs
124+ const specsQuery = `
125+ SELECT
126+ spec_url as full_name,
127+ GROUP_CONCAT(DISTINCT test_run_version) AS versions
128+ FROM TestSpecs
129+ GROUP BY full_name
130+ ORDER BY full_name
131+ ` ;
132+ const specsRows = await all ( specsQuery ) ;
133+ const specs = { } ;
134+ const flatSpecs = { } ;
135+
136+ for ( const row of specsRows ) {
137+ const { versions, full_name } = row ;
138+ let current = full_name ;
139+
140+ while ( current !== "null" ) {
141+ const slug = slugify ( current ) ;
142+ const parent = computeParent ( current ) ;
143+ const { name, isHashed } = computeName ( current )
144+
145+ if ( ! specs [ parent ] ) {
146+ specs [ parent ] = { } ;
147+ }
148+
149+ flatSpecs [ current ] = true
150+
151+ specs [ parent ] [ current ] = {
152+ versions : versions ?. split ( ',' ) || [ ] ,
153+ spec_full_name : current ,
154+ slug,
155+ name,
156+ isHashed,
157+ } ;
158+
159+ current = parent ;
160+ }
161+ }
162+ outputJSON ( "data/specs.json" , specs ) ;
163+
164+ const descendTheSpecsTree = ( current , path ) => {
165+ Object . entries ( specs [ current ] || { } )
166+ . forEach ( ( [ key , spec ] ) => {
167+ const addSpecs = ( current ) => {
168+ let hashes = [ ...( current . specs || [ ] ) , spec . name ] ;
169+ hashes = [ ...new Set ( hashes ) ] ; // deduplicate
170+ return { ...current , hashes }
171+ } ;
172+
173+ // To reproduce the structure of URLs and hashes, we update existing specs pages
174+ if ( spec . isHashed ) {
175+ const p = path . join ( "/" ) ;
176+ outputFrontmatter (
177+ `content/specs/${ p } ,
178+ addSpecs
179+ ) ;
180+ // We assume there are no recursion / children for hashes
181+ return
182+ }
183+
184+ const newPath = [ ...path , spec . name ] ;
185+ const p = newPath . join ( "/" ) ;
186+
187+ outputFrontmatter ( `content/specs/${ p } , {
188+ ...spec ,
189+ title : spec . name ,
190+ } ) ;
191+
192+ descendTheSpecsTree ( key , newPath ) ;
193+ } )
194+ }
195+
196+ descendTheSpecsTree ( "null" , [ ] )
197+
198+ // Aggregate test results per specs
199+ const specsTestGroups = { } ;
200+
201+ for ( const fullName of Object . keys ( flatSpecs ) ) {
202+ // list all the test names for a given spec.
203+ // we prefix search the database for spec_urls starting with the spec name
204+ const specsQuery = `
205+ SELECT
206+ test_full_name
207+ FROM TestSpecs
208+ WHERE spec_url LIKE ?
209+ ORDER BY test_full_name
210+ ` ;
211+ const tests = await all ( specsQuery , [ fullName + '%' ] ) ;
212+
213+ const s = tests . map ( x => x . test_full_name )
214+ . reduce ( ( acc , name ) => {
215+ return {
216+ ...acc ,
217+ [ name ] : flatTestGroups [ name ]
218+ }
219+ } , { } ) ;
220+ specsTestGroups [ fullName ] = s ;
221+ }
222+
223+ outputJSON ( "data/specsgroups.json" , specsTestGroups ) ;
224+
73225 // Query to fetch all stdouts
74226 const logsQuery = `
75227 SELECT
@@ -269,6 +421,7 @@ const slugify = (str) => {
269421 . replace ( / _ + / g, '_' ) // remove consecutive underscores
270422 . replace ( / [ \/ ] / g, "__" )
271423 . replace ( / [ ^ a - z 0 - 9 - ] / g, '-' ) // remove non-alphanumeric characters
424+ . replace ( / - + / g, '-' ) // remove consecutive dashes
272425}
273426
274427const outputJSON = ( p , data ) => {
@@ -283,7 +436,7 @@ const outputJSON = (p, data) => {
283436 fs . writeFileSync ( fullPath , json ) ;
284437}
285438
286- const outputFrontmatter = ( p , data ) => {
439+ const outputFrontmatter = ( p , dataOrUpdate ) => {
287440 const fullPath = `${ hugoOutput } ${ p } ;
288441
289442 // TODO: implement update frontmatter
@@ -303,7 +456,12 @@ const outputFrontmatter = (p, data) => {
303456 content . content = existing . content ;
304457 content . data = existing . data ;
305458 }
306- content . data = { ...content . data , ...data } ;
459+
460+ if ( typeof dataOrUpdate === "function" ) {
461+ content . data = dataOrUpdate ( content . data ) ;
462+ } else {
463+ content . data = { ...content . data , ...dataOrUpdate } ;
464+ }
307465
308466 const md = matter . stringify ( content . content , content . data ) ;
309467 fs . writeFileSync ( fullPath , md ) ;
@@ -322,4 +480,4 @@ main()
322480 . catch ( ( e ) => {
323481 console . error ( e ) ;
324482 process . exit ( 1 ) ;
325- } )
483+ } )
0 commit comments