From ea9cc83e5315d256c80cf107ccd18caacaa6bc1a Mon Sep 17 00:00:00 2001 From: mohamed yahia Date: Wed, 10 Jan 2024 03:18:33 +0200 Subject: [PATCH 1/3] [lib][xl]: Add tests for computed fields and document types, Fix duplicate tags issue (#108) * Throw error when the schema is incorrect * Fix duplicate body tag, Fix linting error * Fix the library incorrectly stringifying strings * Update tags test for duplicated body tags * Add tests for document types * Update test * add test for computed fields * remove console.log * Add changeset * Remove file * Fix test error --- .changeset/dirty-spies-turn.md | 10 ++ package-lock.json | 4 +- src/lib/indexFolder.ts | 13 +- src/lib/parseFile.ts | 7 +- src/lib/schema.ts | 3 +- src/tests/computedField.spec.ts | 286 ++++++++++++++++++++++++++++++++ src/tests/documentTypes.spec.ts | 66 ++++++++ src/tests/markdowndb.spec.ts | 4 +- src/tests/parseFile.spec.ts | 6 - 9 files changed, 379 insertions(+), 20 deletions(-) create mode 100644 .changeset/dirty-spies-turn.md create mode 100644 src/tests/computedField.spec.ts create mode 100644 src/tests/documentTypes.spec.ts diff --git a/.changeset/dirty-spies-turn.md b/.changeset/dirty-spies-turn.md new file mode 100644 index 0000000..115e57b --- /dev/null +++ b/.changeset/dirty-spies-turn.md @@ -0,0 +1,10 @@ +--- +"mddb": patch +--- + +- Add tests for document types +- Fix throwing an error when the document type is incorrect +- Fix linting error +- Fix a strange duplicated body tags issue +- Fix the library incorrectly stringifying strings +- Add tests for computed fields diff --git a/package-lock.json b/package-lock.json index b8da53b..4133ad1 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "mddb", - "version": "0.8.0", + "version": "0.9.1", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "mddb", - "version": "0.8.0", + "version": "0.9.1", "license": "MIT", "dependencies": { "@portaljs/remark-wiki-link": "^1.0.4", diff --git a/src/lib/indexFolder.ts b/src/lib/indexFolder.ts index a909d58..87b16a4 100644 --- a/src/lib/indexFolder.ts +++ b/src/lib/indexFolder.ts @@ -47,13 +47,16 @@ export function indexFolder( const error: ZodError = (result as any).error; error.errors.forEach((err: any) => { - const errorMessage = `Error: In ${ - fileObject.file_path - } for the ${documentType} schema. \n In "${err.path.join( - "," - )}" field: ${err.message}`; + const errorMessage = `Error: In ${fileObject.file_path + } for the ${documentType} schema. \n In "${err.path.join( + "," + )}" field: ${err.message}`; console.error(errorMessage); }); + + throw new Error( + "Validation Failed: Unable to validate files against the specified scheme. Ensure that the file formats and content adhere to the specified scheme." + ); } } diff --git a/src/lib/parseFile.ts b/src/lib/parseFile.ts index 2bdb898..5cc4fdd 100644 --- a/src/lib/parseFile.ts +++ b/src/lib/parseFile.ts @@ -23,6 +23,7 @@ export function parseFile(source: string, options?: ParsingOptions) { // Links const links = extractWikiLinks(ast, options); + metadata.tags = Array.from(new Set(metadata.tags)); const tasks = extractTasks(ast); metadata.tasks = tasks; @@ -78,7 +79,7 @@ export const extractTagsFromBody = (ast: Root) => { function extractTags(text: string) { let tags: any = []; - const textTags = text.match(/(?:^|\s+|\n+|\r+)#([a-zA-Z0-9_\-\/\p{L}]+)/gu); + const textTags = text.match(/(?:^|\s+|\n+|\r+)#([a-zA-Z0-9_\-/\p{L}]+)/gu); if (textTags) { tags = tags.concat( textTags @@ -94,9 +95,9 @@ function isValidTag(tag: string) { // Check if the tag follows the specified rules return ( tag.length > 1 && - /[a-zA-Z_\-\/\p{L}]+/gu.test(tag) && // At least one non-numerical character + /[a-zA-Z_\-/\p{L}]+/gu.test(tag) && // At least one non-numerical character !/\s/.test(tag) && // No blank spaces - /[a-zA-Z0-9_\-\/\p{L}]+/gu.test(tag) // Valid characters: alphabetical letters, numbers, underscore, hyphen, forward slash, and any letter in any language + /[a-zA-Z0-9_\-/\p{L}]+/gu.test(tag) // Valid characters: alphabetical letters, numbers, underscore, hyphen, forward slash, and any letter in any language ); } diff --git a/src/lib/schema.ts b/src/lib/schema.ts index a0267a3..0fe3615 100644 --- a/src/lib/schema.ts +++ b/src/lib/schema.ts @@ -112,7 +112,8 @@ class MddbFile { // If the value is undefined, default it to null if (value !== undefined) { const shouldStringify = - key === "metadata" || !MddbFile.defaultProperties.includes(key); + (key === "metadata" || !MddbFile.defaultProperties.includes(key)) && + typeof value === "object"; // Stringify all user-defined fields and metadata serializedFile[key] = shouldStringify ? JSON.stringify(value) : value; } else { diff --git a/src/tests/computedField.spec.ts b/src/tests/computedField.spec.ts new file mode 100644 index 0000000..d067f3f --- /dev/null +++ b/src/tests/computedField.spec.ts @@ -0,0 +1,286 @@ +import { FileInfo, processFile } from "../lib/process"; +import Path from "path"; +import { Root, Content } from "mdast"; + +type MyContent = Content & { + children?: { value: string }[]; +}; + +describe("Can parse a file and get file info", () => { + const pathToContentFixture = "__mocks__/content"; + const filePath = "index.mdx"; + const fullPath = Path.join(pathToContentFixture, filePath); + test("Can get some values from AST and Add it to fileInfo", async () => { + const fileInfo = processFile( + pathToContentFixture, + fullPath, + (filePath) => filePath, + [], + [ + (fileInfo: FileInfo, ast: Root) => { + const headingNode = ast.children.filter((child) => { + return child.type === "heading" && child.depth === 1; + }) as MyContent[]; + fileInfo.title = headingNode[0]?.children[0]?.value; + }, + ] + ); + + expect(fileInfo.file_path).toBe(fullPath); + expect(fileInfo.url_path).toBe("index.mdx"); + expect(fileInfo.title).toBe("Welcome"); + expect(fileInfo.extension).toBe("mdx"); + + expect(fileInfo.tags).toEqual([ + "tag1", + "tag2", + "tag3", + "日本語タグ", + "标签名", + "метка", + "태그이름", + "tag_فارسی", + "Tag_avec_éèç-_öäüßñ", + ]); + + expect(fileInfo.metadata).toEqual({ + title: "Homepage", + tags: [ + "tag1", + "tag2", + "tag3", + "日本語タグ", + "标签名", + "метка", + "태그이름", + "tag_فارسی", + "Tag_avec_éèç-_öäüßñ", + ], + tasks: [ + { + checked: false, + description: "uncompleted task 2", + }, + { + checked: true, + description: "completed task 1", + }, + { + checked: true, + description: "completed task 2", + }, + ], + }); + expect(fileInfo.links).toEqual([ + { + embed: false, + from: "index.mdx", + internal: true, + text: "link", + to: "blog0.mdx", + toRaw: "blog0.mdx", + }, + ]); + }); + test("Can add array,object,null,undefined values throw the computed fields", () => { + const fileInfo = processFile( + pathToContentFixture, + fullPath, + (filePath) => filePath, + [], + [ + // add homepage string to file info + (fileInfo: FileInfo, ast: Root) => { + fileInfo.public = { + title: "Title", + pageSize: 34, + isLocked: false, + }; + }, + // add pageNumber as number to fileInfo + (fileInfo: FileInfo, ast: Root) => { + fileInfo.Authors = ["Abdelrhiim", "Mohammed", "John"]; + }, + // add isLocked as boolean to fileInfo + (fileInfo: FileInfo, ast: Root) => { + fileInfo.matrix = null; + }, + // add isLocked as boolean to fileInfo + (fileInfo: FileInfo, ast: Root) => { + fileInfo.building = undefined; + }, + ] + ); + expect(fileInfo.file_path).toBe(fullPath); + expect(fileInfo.url_path).toBe("index.mdx"); + expect(fileInfo.public).toEqual({ + title: "Title", + pageSize: 34, + isLocked: false, + }); + expect(fileInfo.Authors).toEqual(["Abdelrhiim", "Mohammed", "John"]); + expect(fileInfo.matrix).toBeNull(); + expect(fileInfo.building).toBeUndefined(); + }); + test("Can add string,number,and boolean values throw the computed fields", () => { + const fileInfo = processFile( + pathToContentFixture, + fullPath, + (filePath) => filePath, + [], + [ + // add homepage string to file info + (fileInfo: FileInfo, ast: Root) => { + fileInfo.homePage = "indexFile"; + }, + // add pageNumber as number to fileInfo + (fileInfo: FileInfo, ast: Root) => { + fileInfo.pageNumber = 23; + }, + // add isLocked as boolean to fileInfo + (fileInfo: FileInfo, ast: Root) => { + fileInfo.isLocked = false; + }, + ] + ); + expect(fileInfo.file_path).toBe(fullPath); + expect(fileInfo.url_path).toBe("index.mdx"); + expect(fileInfo.homePage).toBe("indexFile"); + expect(fileInfo.pageNumber).toBe(23); + expect(fileInfo.isLocked).toBe(false); + }); + test("Can add metadata field threw the computed fields", async () => { + const fileInfo = processFile( + pathToContentFixture, + fullPath, + (filePath) => filePath, + [], + [ + (fileInfo: FileInfo, ast: Root) => { + fileInfo.metadata.AuthorName = "John Smith"; + }, + ] + ); + + expect(fileInfo.file_path).toBe(fullPath); + expect(fileInfo.url_path).toBe("index.mdx"); + expect(fileInfo.extension).toBe("mdx"); + + expect(fileInfo.tags).toEqual([ + "tag1", + "tag2", + "tag3", + "日本語タグ", + "标签名", + "метка", + "태그이름", + "tag_فارسی", + "Tag_avec_éèç-_öäüßñ", + ]); + expect(fileInfo.metadata).toEqual({ + AuthorName: "John Smith", + title: "Homepage", + tags: [ + "tag1", + "tag2", + "tag3", + "日本語タグ", + "标签名", + "метка", + "태그이름", + "tag_فارسی", + "Tag_avec_éèç-_öäüßñ", + ], + tasks: [ + { + checked: false, + description: "uncompleted task 2", + }, + { + checked: true, + description: "completed task 1", + }, + { + checked: true, + description: "completed task 2", + }, + ], + }); + expect(fileInfo.links).toEqual([ + { + embed: false, + from: "index.mdx", + internal: true, + text: "link", + to: "blog0.mdx", + toRaw: "blog0.mdx", + }, + ]); + }); + test("Can Edit and Delete Metadata Field threw the computed fields", () => { + const fileInfo = processFile( + pathToContentFixture, + fullPath, + (filePath) => filePath, + [], + [ + (fileInfo: FileInfo, ast: Root) => { + fileInfo.metadata.title = "Second Page"; + delete fileInfo.metadata.AuthorName; + }, + ] + ); + expect(fileInfo.file_path).toBe(fullPath); + expect(fileInfo.url_path).toBe("index.mdx"); + expect(fileInfo.extension).toBe("mdx"); + expect(fileInfo.tags).toEqual([ + "tag1", + "tag2", + "tag3", + "日本語タグ", + "标签名", + "метка", + "태그이름", + "tag_فارسی", + "Tag_avec_éèç-_öäüßñ", + ]); + expect(fileInfo.metadata).toEqual({ + title: "Second Page", + tags: [ + "tag1", + "tag2", + "tag3", + "日本語タグ", + "标签名", + "метка", + "태그이름", + "tag_فارسی", + "Tag_avec_éèç-_öäüßñ", + ], + tasks: [ + { + checked: false, + description: "uncompleted task 2", + }, + { + checked: true, + description: "completed task 1", + }, + { + checked: true, + description: "completed task 2", + }, + ], + }); + expect(fileInfo.links).toEqual([ + { + embed: false, + from: "index.mdx", + internal: true, + text: "link", + to: "blog0.mdx", + toRaw: "blog0.mdx", + }, + ]); + }); +}); diff --git a/src/tests/documentTypes.spec.ts b/src/tests/documentTypes.spec.ts new file mode 100644 index 0000000..cdca9eb --- /dev/null +++ b/src/tests/documentTypes.spec.ts @@ -0,0 +1,66 @@ +import { FileInfo } from "../lib/process"; +import { Root } from "mdast"; +import { MarkdownDB } from "../lib/markdowndb"; +import { z } from "zod"; + +describe("Document Types Schema Validate Testing", () => { + const pathToContentFixture = "__mocks__/content"; + let mddb: MarkdownDB; + + beforeAll(async () => { + const dbConfig = { + client: "sqlite3", + connection: { + filename: "markdown.db", + }, + }; + mddb = new MarkdownDB(dbConfig); + await mddb.init(); + }); + + afterAll(async () => { + await mddb.db.destroy(); + }); + + test("Test that the 'indexFolder' function throws a validation error for a missing field in the blog schema.", async () => { + try { + await mddb.indexFolder({ + folderPath: pathToContentFixture, + customConfig: { + schemas: { + blog: z.object({ + missingField: z.string(), + }), + }, + }, + }); + + fail("Expected a validation error due to a missing field in the blog schema, but none was thrown."); + } catch (error: any) { + expect(error).toBeInstanceOf(Error); + expect(error.message).toContain("Validation Failed: Unable to validate files against the specified scheme."); + ; + } + }); + test("Should check if the title field is created and save in db", async () => { + await mddb.indexFolder({ + folderPath: pathToContentFixture, + customConfig: { + computedFields: [ + (fileInfo: FileInfo, ast: Root) => { + fileInfo.title = "Hello"; + }, + ], + schemas: { + blog: z.object({ + title: z.string(), + }), + }, + }, + }); + const dbFiles = await mddb.getFiles({ filetypes: ["blog"] }); + for (const file of dbFiles) { + expect(file.title).toBe("Hello"); + } + }); +}); diff --git a/src/tests/markdowndb.spec.ts b/src/tests/markdowndb.spec.ts index 4cae13b..c26c499 100644 --- a/src/tests/markdowndb.spec.ts +++ b/src/tests/markdowndb.spec.ts @@ -27,9 +27,7 @@ describe("MarkdownDB - default config", () => { }); afterAll(async () => { - // TODO why we have to call this twice? - mddb.db.destroy(); - mddb._destroyDb(); + await mddb.db.destroy(); }); describe("correct startup and indexing", () => { diff --git a/src/tests/parseFile.spec.ts b/src/tests/parseFile.spec.ts index 72bd659..360a0bb 100644 --- a/src/tests/parseFile.spec.ts +++ b/src/tests/parseFile.spec.ts @@ -109,12 +109,6 @@ describe("parseFile", () => { "태그이름", "tag_فارسی", "Tag_avec_éèç-_öäüßñ", - "日本語タグ", - "标签名", - "метка", - "태그이름", - "tag_فارسی", - "Tag_avec_éèç-_öäüßñ", ], tasks: [ { description: "uncompleted task", checked: false }, From 86bc39ad75fe2d8ecaa4bf6f33c760c7a9cc26d8 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Mon, 15 Jan 2024 22:33:25 +0200 Subject: [PATCH 2/3] Version Packages (#109) Co-authored-by: github-actions[bot] --- .changeset/dirty-spies-turn.md | 10 ---------- CHANGELOG.md | 11 +++++++++++ package.json | 2 +- 3 files changed, 12 insertions(+), 11 deletions(-) delete mode 100644 .changeset/dirty-spies-turn.md diff --git a/.changeset/dirty-spies-turn.md b/.changeset/dirty-spies-turn.md deleted file mode 100644 index 115e57b..0000000 --- a/.changeset/dirty-spies-turn.md +++ /dev/null @@ -1,10 +0,0 @@ ---- -"mddb": patch ---- - -- Add tests for document types -- Fix throwing an error when the document type is incorrect -- Fix linting error -- Fix a strange duplicated body tags issue -- Fix the library incorrectly stringifying strings -- Add tests for computed fields diff --git a/CHANGELOG.md b/CHANGELOG.md index bdcf19f..4b64bc1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,16 @@ # mddb +## 0.9.2 + +### Patch Changes + +- [#108](https://github.com/datopian/markdowndb/pull/108) [`ea9cc83`](https://github.com/datopian/markdowndb/commit/ea9cc83e5315d256c80cf107ccd18caacaa6bc1a) Thanks [@mohamedsalem401](https://github.com/mohamedsalem401)! - - Add tests for document types + - Fix throwing an error when the document type is incorrect + - Fix linting error + - Fix a strange duplicated body tags issue + - Fix the library incorrectly stringifying strings + - Add tests for computed fields + ## 0.9.1 ### Patch Changes diff --git a/package.json b/package.json index 0050788..b2cba54 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "mddb", - "version": "0.9.1", + "version": "0.9.2", "description": "Parse markdown files and store them in an SQL database.", "bin": { "mddb": "./dist/src/bin/index.js" From 1b3c70df1b1ccd2e200c1c0480feb7c92d8524ac Mon Sep 17 00:00:00 2001 From: Rufus Pollock Date: Tue, 30 Jan 2024 09:32:12 +0100 Subject: [PATCH 3/3] [README][xs]: update link to discord server. --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index e32dbd8..fda14ca 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,7 @@ # MarkdownDB [![](https://badgen.net/npm/v/mddb)](https://www.npmjs.com/package/mddb) -[![](https://dcbadge.vercel.app/api/server/EeyfGrGu4U)](https://discord.gg/EeyfGrGu4U) +[![](https://dcbadge.vercel.app/api/server/xfFDMPU9dC)](https://discord.gg/xfFDMPU9dC) MarkdownDB is a javascript library that turns markdown files into structured queryable databaase (SQL-based and simple JSON). It helps you build rich markdown-powered sites easily and reliably. Specifically it: