Skip to content

Commit 3531779

Browse files
dario-piotrowiczdario-piotrowicz
authored
chore: refactor the generateBlogData function (again) (#7607) (#7618)
* chore: refactor the `generateBlogData` function (#7607) * add unit tests for `generateBlogData` * avoid mocking node:path * remove original modules * migrate to node test runner
1 parent 0b24039 commit 3531779

File tree

2 files changed

+261
-55
lines changed

2 files changed

+261
-55
lines changed

‎apps/site/next-data/generators/__tests__/blogData.test.mjs

+209
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,209 @@
1+
import assert from 'node:assert/strict';
2+
import { normalize } from 'node:path';
3+
import { Readable } from 'node:stream';
4+
import { describe, it, mock } from 'node:test';
5+
6+
let files = [];
7+
8+
mock.module('node:fs', {
9+
namedExports: {
10+
createReadStream: filename => {
11+
const readable = new Readable();
12+
const file = files.find(f => filename.endsWith(normalize(f.path)));
13+
readable.push(`---\n`);
14+
file.frontMatterContent.forEach(line => readable.push(`${line}\n`));
15+
readable.push(`---\n`);
16+
readable.push(null);
17+
readable.close = () => {};
18+
return readable;
19+
},
20+
},
21+
});
22+
23+
mock.module('../../../next.helpers.mjs', {
24+
namedExports: {
25+
getMarkdownFiles: () => {
26+
return Promise.resolve(files.map(file => file.path));
27+
},
28+
},
29+
});
30+
31+
const generateBlogData = (await import('../blogData.mjs')).default;
32+
33+
describe('generateBlogData', () => {
34+
it('should return zero posts and only the default "all" category if no md file is found', async () => {
35+
files = [];
36+
37+
const blogData = await generateBlogData();
38+
39+
assert.deepEqual(blogData.categories, ['all']);
40+
assert.deepEqual(blogData.posts, []);
41+
});
42+
43+
it('should collect the data from a single md file if only one is found', async () => {
44+
files = [
45+
{
46+
path: 'pages/en/blog/post1.md',
47+
frontMatterContent: [
48+
`date: '2020-01-01T00:00:00.000Z'`,
49+
`title: POST 1`,
50+
`author: author`,
51+
],
52+
},
53+
];
54+
55+
const blogData = await generateBlogData();
56+
57+
assert.equal(blogData.posts.length, 1);
58+
const post = blogData.posts[0];
59+
assert.equal(post.title, 'POST 1');
60+
assert.deepEqual(post.date, new Date('2020-01-01T00:00:00.000Z'));
61+
assert.equal(post.author, 'author');
62+
});
63+
64+
it('should collect the data from a single md file if only one is found', async () => {
65+
files = [
66+
{
67+
path: 'pages/en/blog/post1.md',
68+
frontMatterContent: [
69+
`date: '2020-01-01T00:00:00.000Z'`,
70+
`title: POST 1`,
71+
`author: author`,
72+
],
73+
},
74+
];
75+
76+
const blogData = await generateBlogData();
77+
78+
assert.equal(blogData.posts.length, 1);
79+
const post = blogData.posts[0];
80+
assert.equal(post.title, 'POST 1');
81+
assert.deepEqual(post.date, new Date('2020-01-01T00:00:00.000Z'));
82+
assert.equal(post.author, 'author');
83+
});
84+
85+
it('should collect the data from multiple md files', async () => {
86+
const currentDate = new Date();
87+
88+
files = [
89+
{
90+
path: 'pages/en/blog/post1.md',
91+
frontMatterContent: [
92+
`date: '2020-01-01T00:00:00.000Z'`,
93+
`title: POST 1`,
94+
`author: author-a`,
95+
],
96+
},
97+
{
98+
path: 'pages/en/blog/post2.md',
99+
frontMatterContent: [
100+
`date: '2020-01-02T00:00:00.000Z'`,
101+
`title: POST 2`,
102+
`author: author-b`,
103+
],
104+
},
105+
{
106+
path: 'pages/en/blog/post3.md',
107+
frontMatterContent: [
108+
// no date specified (the date defaults to the current date)
109+
`title: POST 3`,
110+
`author: author-c`,
111+
],
112+
},
113+
];
114+
115+
const blogData = await generateBlogData();
116+
117+
assert(blogData.posts.length, 3);
118+
assert.equal(blogData.posts[0].title, 'POST 1');
119+
assert.deepEqual(
120+
blogData.posts[0].date,
121+
new Date('2020-01-01T00:00:00.000Z')
122+
);
123+
assert.equal(blogData.posts[0].author, 'author-a');
124+
assert.equal(blogData.posts[1].title, 'POST 2');
125+
assert.deepEqual(
126+
blogData.posts[1].date,
127+
new Date('2020-01-02T00:00:00.000Z')
128+
);
129+
assert.equal(blogData.posts[1].author, 'author-b');
130+
assert.equal(blogData.posts[2].title, 'POST 3');
131+
assert.equal(
132+
blogData.posts[2].date.setMilliseconds(0),
133+
currentDate.setMilliseconds(0)
134+
);
135+
assert.equal(blogData.posts[2].author, 'author-c');
136+
});
137+
138+
it('should generate categories based on the categories of md files and their years', async () => {
139+
files = [
140+
{
141+
path: 'pages/en/blog/post1.md',
142+
frontMatterContent: [
143+
"date: '2020-01-01T00:00:00.000Z'",
144+
'category: category-a',
145+
],
146+
},
147+
{
148+
path: 'pages/en/blog/sub-dir/post2.md',
149+
frontMatterContent: [
150+
"date: '2020-01-02T00:00:00.000Z'",
151+
'category: category-b',
152+
],
153+
},
154+
{
155+
path: 'pages/en/blog/post3.md',
156+
frontMatterContent: [
157+
"date: '2021-03-13T00:00:00.000Z'",
158+
// no category specified (it should be "uncategorized")
159+
],
160+
},
161+
{
162+
path: 'pages/en/blog/post4.md',
163+
frontMatterContent: [
164+
// no date specified (the date defaults to the current date)
165+
'category: category-b',
166+
],
167+
},
168+
];
169+
170+
const blogData = await generateBlogData();
171+
172+
assert.deepEqual(blogData.categories.sort(), [
173+
'all',
174+
'category-a',
175+
'category-b',
176+
'uncategorized',
177+
'year-2020',
178+
'year-2021',
179+
`year-${new Date().getUTCFullYear()}`,
180+
]);
181+
});
182+
183+
it('should generate slugs based on the md filenames and categories', async () => {
184+
files = [
185+
{
186+
path: 'pages/en/blog/post1.md',
187+
frontMatterContent: ['category: category-a'],
188+
},
189+
{
190+
path: 'pages/en/blog/post2.md',
191+
frontMatterContent: ['category: category-b'],
192+
},
193+
{
194+
path: 'pages/en/blog/post3.md',
195+
frontMatterContent: [
196+
// no category specified
197+
],
198+
},
199+
];
200+
201+
const blogData = await generateBlogData();
202+
203+
assert.deepEqual(blogData.posts.map(p => p.slug).sort(), [
204+
'/blog/category-a/post1',
205+
'/blog/category-b/post2',
206+
'/blog/uncategorized/post3',
207+
]);
208+
});
209+
});

‎apps/site/next-data/generators/blogData.mjs

+52-55
Original file line numberDiff line numberDiff line change
@@ -11,11 +11,6 @@ import { getMarkdownFiles } from '../../next.helpers.mjs';
1111
// gets the current blog path based on local module path
1212
const blogPath = join(process.cwd(), 'pages/en/blog');
1313

14-
/**
15-
* This contains the metadata of all available blog categories
16-
*/
17-
const blogCategories = new Set(['all']);
18-
1914
/**
2015
* This method parses the source (raw) Markdown content into Frontmatter
2116
* and returns basic information for blog posts
@@ -39,12 +34,6 @@ const getFrontMatter = (filename, source) => {
3934
// all = (all blog posts), publish year and the actual blog category
4035
const categories = [category, `year-${publishYear}`, 'all'];
4136

42-
// we add the year to the categories set
43-
blogCategories.add(`year-${publishYear}`);
44-
45-
// we add the category to the categories set
46-
blogCategories.add(category);
47-
4837
// this is the url used for the blog post it based on the category and filename
4938
const slug = `/blog/${category}/${basename(filename, extname(filename))}`;
5039

@@ -63,50 +52,58 @@ const generateBlogData = async () => {
6352
'**/index.md',
6453
]);
6554

66-
return new Promise(resolve => {
67-
const posts = [];
68-
const rawFrontmatter = [];
69-
70-
filenames.forEach(filename => {
71-
// We create a stream for reading a file instead of reading the files
72-
const _stream = createReadStream(join(blogPath, filename));
73-
74-
// We create a readline interface to read the file line-by-line
75-
const _readLine = readline.createInterface({ input: _stream });
76-
77-
// Creates an array of the metadata based on the filename
78-
// This prevents concurrency issues since the for-loop is synchronous
79-
// and these event listeners are not
80-
rawFrontmatter[filename] = [0, ''];
81-
82-
// We read line by line
83-
_readLine.on('line', line => {
84-
rawFrontmatter[filename][1] += `${line}\n`;
85-
86-
// We observe the frontmatter separators
87-
if (line === '---') {
88-
rawFrontmatter[filename][0] += 1;
89-
}
90-
91-
// Once we have two separators we close the readLine and the stream
92-
if (rawFrontmatter[filename][0] === 2) {
93-
_readLine.close();
94-
_stream.close();
95-
}
96-
});
97-
98-
// Then we parse gray-matter on the frontmatter
99-
// This allows us to only read the frontmatter part of each file
100-
// and optimise the read-process as we have thousands of markdown files
101-
_readLine.on('close', () => {
102-
posts.push(getFrontMatter(filename, rawFrontmatter[filename][1]));
103-
104-
if (posts.length === filenames.length) {
105-
resolve({ categories: [...blogCategories], posts });
106-
}
107-
});
108-
});
109-
});
55+
/**
56+
* This contains the metadata of all available blog categories
57+
*/
58+
const blogCategories = new Set(['all']);
59+
60+
const posts = await Promise.all(
61+
filenames.map(
62+
filename =>
63+
new Promise(resolve => {
64+
// We create a stream for reading a file instead of reading the files
65+
const _stream = createReadStream(join(blogPath, filename));
66+
67+
// We create a readline interface to read the file line-by-line
68+
const _readLine = readline.createInterface({ input: _stream });
69+
70+
let rawFrontmatter = '';
71+
let frontmatterSeparatorsEncountered = 0;
72+
73+
// We read line by line
74+
_readLine.on('line', line => {
75+
rawFrontmatter += `${line}\n`;
76+
77+
// We observe the frontmatter separators
78+
if (line === '---') {
79+
frontmatterSeparatorsEncountered++;
80+
}
81+
82+
// Once we have two separators we close the readLine and the stream
83+
if (frontmatterSeparatorsEncountered === 2) {
84+
_readLine.close();
85+
_stream.close();
86+
}
87+
});
88+
89+
// Then we parse gray-matter on the frontmatter
90+
// This allows us to only read the frontmatter part of each file
91+
// and optimise the read-process as we have thousands of markdown files
92+
_readLine.on('close', () => {
93+
const frontMatterData = getFrontMatter(filename, rawFrontmatter);
94+
95+
frontMatterData.categories.forEach(category => {
96+
// we add the category to the categories set
97+
blogCategories.add(category);
98+
});
99+
100+
resolve(frontMatterData);
101+
});
102+
})
103+
)
104+
);
105+
106+
return { categories: [...blogCategories], posts };
110107
};
111108

112109
export default generateBlogData;

0 commit comments

Comments
 (0)