From 6cb668f47dce54784ff79cc576854ec91ca3ebf1 Mon Sep 17 00:00:00 2001 From: mohamed yahia Date: Thu, 7 Mar 2024 17:05:22 +0200 Subject: [PATCH] Parse task inline fields (#115) * Added code and test for parsing task inline fields. Code is not working Added devcontainer.json for VSCode development environment. * Added function to parse inline fields in tasks. * Updated Task interface to have fields from obsidian dataview task metadata and updated tests. * Task table is being created, tests are still passing. No new DB specific tests yet. * Added tags to the mock metadata document. * remove devcontainer --------- Co-authored-by: David Stenglein Co-authored-by: David Stenglein --- __mocks__/content/taskmetadata.md | 6 ++ src/lib/databaseUtils.ts | 20 +++++- src/lib/markdowndb.ts | 6 +- src/lib/parseFile.ts | 51 ++++++++++++--- src/lib/process.ts | 6 +- src/lib/schema.ts | 73 ++++++++++++++++++++- src/tests/computedField.spec.ts | 55 ++++++++++++++++ src/tests/extractTasks.spec.ts | 101 ++++++++++++++++++++++++++---- src/tests/parseFile.spec.ts | 70 +++++++++++++++++++-- src/tests/process.spec.ts | 18 ++++++ 10 files changed, 379 insertions(+), 27 deletions(-) create mode 100644 __mocks__/content/taskmetadata.md diff --git a/__mocks__/content/taskmetadata.md b/__mocks__/content/taskmetadata.md new file mode 100644 index 0000000..880e441 --- /dev/null +++ b/__mocks__/content/taskmetadata.md @@ -0,0 +1,6 @@ +--- +title: Task metadata fixture +--- + +- [ ] Task without metadata +- [x] Task with metadata #tag1 #tag2 [person:: Athena Person] [due:: 2024-10-01] #tag3 \ No newline at end of file diff --git a/src/lib/databaseUtils.ts b/src/lib/databaseUtils.ts index d1c1b65..b4f4c20 100644 --- a/src/lib/databaseUtils.ts +++ b/src/lib/databaseUtils.ts @@ -1,10 +1,10 @@ import { Knex } from "knex"; -import { MddbFile, MddbTag, MddbLink, MddbFileTag, File } from "./schema.js"; +import { MddbFile, MddbTag, MddbTask, MddbLink, MddbFileTag, File } from "./schema.js"; import path from "path"; import { WikiLink } from "./parseFile.js"; export async function resetDatabaseTables(db: Knex) { - const tableNames = [MddbTag, MddbFileTag, MddbLink]; + const tableNames = [MddbTag, MddbFileTag, MddbLink, MddbTask]; // Drop and Create tables for (const table of tableNames) { await table.deleteTable(db); @@ -81,3 +81,19 @@ export function getUniqueProperties(objects: any[]): string[] { return uniqueProperties; } + +export function mapTasksToInsert(file: any) { + return file.tasks.map((task: any) => { + return { + file: file._id, + description: task.description, + checked: task.checked, + metadata: JSON.stringify(task.metadata), + created: task.created, + due: task.due, + completion: task.completion, + start: task.start, + scheduled: task.scheduled, + }; + }); +} diff --git a/src/lib/markdowndb.ts b/src/lib/markdowndb.ts index c017426..bd80ec8 100644 --- a/src/lib/markdowndb.ts +++ b/src/lib/markdowndb.ts @@ -1,7 +1,7 @@ import path from "path"; import knex, { Knex } from "knex"; -import { MddbFile, MddbTag, MddbLink, MddbFileTag } from "./schema.js"; +import { MddbFile, MddbTag, MddbLink, MddbFileTag, MddbTask } from "./schema.js"; import { indexFolder, shouldIncludeFile } from "./indexFolder.js"; import { resetDatabaseTables, @@ -11,6 +11,7 @@ import { mapFileTagsToInsert, getUniqueValues, getUniqueProperties, + mapTasksToInsert, } from "./databaseUtils.js"; import fs from "fs"; import { CustomConfig } from "./CustomConfig.js"; @@ -178,11 +179,14 @@ export class MarkdownDB { .filter(isLinkToDefined); const fileTagsToInsert = fileObjects.flatMap(mapFileTagsToInsert); + const tasksToInsert = fileObjects.flatMap(mapTasksToInsert); + writeJsonToFile(".markdowndb/files.json", fileObjects); await MddbFile.batchInsert(this.db, filesToInsert); await MddbTag.batchInsert(this.db, tagsToInsert); await MddbFileTag.batchInsert(this.db, fileTagsToInsert); await MddbLink.batchInsert(this.db, getUniqueValues(linksToInsert)); + await MddbTask.batchInsert(this.db, tasksToInsert); } /** diff --git a/src/lib/parseFile.ts b/src/lib/parseFile.ts index 5cc4fdd..a710be5 100644 --- a/src/lib/parseFile.ts +++ b/src/lib/parseFile.ts @@ -6,6 +6,7 @@ import * as path from "path"; import gfm from "remark-gfm"; import remarkWikiLink from "@portaljs/remark-wiki-link"; import { Root } from "remark-parse/lib"; +import { MetaData, Task } from "./schema"; export function parseFile(source: string, options?: ParsingOptions) { // Metadata @@ -189,22 +190,30 @@ export const extractWikiLinks = (ast: Root, options?: ParsingOptions) => { return wikiLinks; }; -export interface Task { - description: string; - checked: boolean; -} - export const extractTasks = (ast: Root) => { const nodes = selectAll("*", ast); const tasks: Task[] = []; nodes.map((node: any) => { if (node.type === "listItem") { const description = recursivelyExtractText(node).trim(); - const checked = node.checked; - if (checked !== null && checked !== undefined) { + const metadata = extractAllTaskMetadata(description); + const checked = node.checked !== null && node.checked !== undefined ? node.checked : null; + const created = metadata.created !== null && metadata.created !== undefined ? metadata.created : null; + const due = metadata.due !== null && metadata.due !== undefined ? metadata.due : null; + const completion = metadata.completion !== null && metadata.completion !== undefined ? metadata.completion : null; + const scheduled = metadata.scheduled !== null && metadata.scheduled !== undefined ? metadata.scheduled : null; + const start = metadata.start !== null && metadata.start !== undefined ? metadata.start : null; + + if (checked !== null) { tasks.push({ description, checked, + created, + due, + completion, + scheduled, + start, + metadata: metadata, }); } } @@ -221,6 +230,34 @@ function recursivelyExtractText(node: any) { } else { return ""; } +}; + +export function extractAllTaskMetadata(description: string) : MetaData { + // Extract metadata fields from the description with the form [field:: value] + // where field is the name of the metadata without spaces and value is the value of the metadata + // There can be multiple metadata fields in the description + const metadataRegex = /\[(.*?)::(.*?)\]/g; + const matches = description.match(metadataRegex); + if (matches) { + const metadata: MetaData = {}; + matches.forEach((match) => { + // extract field and value from groups in the match + const allMatches = match.matchAll(metadataRegex).next().value; + const field = allMatches[1].trim(); + const value = allMatches[2].trim(); + metadata[field] = value; + }); // Add closing parenthesis here + const tags = extractTags(description); + metadata["tags"] = tags; + return metadata; + } else { + return {}; + } + + + + + } // links = extractWikiLinks({ diff --git a/src/lib/process.ts b/src/lib/process.ts index 677262c..4d7e41e 100644 --- a/src/lib/process.ts +++ b/src/lib/process.ts @@ -2,13 +2,14 @@ import crypto from "crypto"; import fs from "fs"; import path from "path"; -import { File } from "./schema.js"; +import { File, Task } from "./schema.js"; import { WikiLink, parseFile } from "./parseFile.js"; import { Root } from "remark-parse/lib/index.js"; export interface FileInfo extends File { tags: string[]; links: WikiLink[]; + tasks: Task[]; } // this file is an extraction of the file info parsing from markdowndb.ts without any sql stuff @@ -39,6 +40,7 @@ export function processFile( metadata: {}, tags: [], links: [], + tasks: [], }; // if not a file type we can parse exit here ... @@ -72,5 +74,7 @@ export function processFile( customFieldFunction(fileInfo, ast); } + fileInfo.tasks = metadata?.tasks || []; + return fileInfo; } diff --git a/src/lib/schema.ts b/src/lib/schema.ts index 0fe3615..454c7c7 100644 --- a/src/lib/schema.ts +++ b/src/lib/schema.ts @@ -9,6 +9,7 @@ export enum Table { Tags = "tags", FileTags = "file_tags", Links = "links", + Tasks = "tasks", } type MetaData = { @@ -308,4 +309,74 @@ class MddbFileTag { } } -export { File, MddbFile, Link, MddbLink, Tag, MddbTag, FileTag, MddbFileTag }; +interface Task { + description: string; + checked: boolean; + due: string | null; + completion: string | null; + created: string; + start: string | null; + scheduled: string | null; + metadata: MetaData | null; + +} + +class MddbTask { + static table = Table.Tasks; + description: string; + checked: boolean; + due: string | null; + completion: string | null; + created: string; + start: string | null; + scheduled: string | null; + metadata: MetaData | null; + + constructor(task: Task) { + this.description = task.description; + this.checked = task.checked; + this.due = task.due; + this.completion = task.completion; + this.created = task.created; + this.start = task.start; + this.scheduled = task.scheduled; + this.metadata = task.metadata; + } + + static async createTable(db: Knex) { + const creator = (table: Knex.TableBuilder) => { + table.string("description").notNullable(); + table.boolean("checked").notNullable(); + table.string("file").notNullable(); + table.string("due"); + table.string("completion"); + table.string("created"); + table.string("start"); + table.string("scheduled"); + table.string("metadata"); + }; + const tableExists = await db.schema.hasTable(this.table); + + if (!tableExists) { + await db.schema.createTable(this.table, creator); + } + } + + static async deleteTable(db: Knex) { + await db.schema.dropTableIfExists(this.table); + } + + static batchInsert(db: Knex, tasks: Task[]) { + if (tasks.length >= 500) { + const promises = []; + for (let i = 0; i < tasks.length; i += 500) { + promises.push(db.batchInsert(Table.Tasks, tasks.slice(i, i + 500))); + } + return Promise.all(promises); + } else { + return db.batchInsert(Table.Tasks, tasks); + } + } +} + +export { MetaData, File, MddbFile, Link, MddbLink, Tag, MddbTag, FileTag, MddbFileTag, Task, MddbTask }; diff --git a/src/tests/computedField.spec.ts b/src/tests/computedField.spec.ts index d067f3f..ef42ac7 100644 --- a/src/tests/computedField.spec.ts +++ b/src/tests/computedField.spec.ts @@ -60,14 +60,32 @@ describe("Can parse a file and get file info", () => { { checked: false, description: "uncompleted task 2", + metadata: {}, + created: null, + due: null, + completion: null, + start: null, + scheduled: null, }, { checked: true, description: "completed task 1", + metadata: {}, + created: null, + due: null, + completion: null, + start: null, + scheduled: null, }, { checked: true, description: "completed task 2", + metadata: {}, + created: null, + due: null, + completion: null, + start: null, + scheduled: null, }, ], }); @@ -195,14 +213,32 @@ describe("Can parse a file and get file info", () => { { checked: false, description: "uncompleted task 2", + metadata: {}, + created: null, + due: null, + completion: null, + start: null, + scheduled: null, }, { checked: true, description: "completed task 1", + metadata: {}, + created: null, + due: null, + completion: null, + start: null, + scheduled: null, }, { checked: true, description: "completed task 2", + metadata: {}, + created: null, + due: null, + completion: null, + start: null, + scheduled: null, }, ], }); @@ -261,14 +297,33 @@ describe("Can parse a file and get file info", () => { { checked: false, description: "uncompleted task 2", + metadata: {}, + created: null, + due: null, + completion: null, + start: null, + scheduled: null, + }, { checked: true, description: "completed task 1", + metadata: {}, + created: null, + due: null, + completion: null, + start: null, + scheduled: null, }, { checked: true, description: "completed task 2", + metadata: {}, + created: null, + due: null, + completion: null, + start: null, + scheduled: null, }, ], }); diff --git a/src/tests/extractTasks.spec.ts b/src/tests/extractTasks.spec.ts index cc3de8e..d1d54f8 100644 --- a/src/tests/extractTasks.spec.ts +++ b/src/tests/extractTasks.spec.ts @@ -8,11 +8,17 @@ const getTasksFromSource = (source: string) => { describe("extractTasks", () => { test("should extract uncompleted tasks from body", () => { + // TODO: Figure out why task 1 is ignored const tasks = getTasksFromSource( "- [] uncompleted task 1\n- [ ] uncompleted task 2" ); const expectedTasks = [ - { description: "uncompleted task 2", checked: false }, + { description: "uncompleted task 2", checked: false, metadata: {}, + created: null, + due: null, + completion: null, + start: null, + scheduled: null, }, ]; expect(tasks).toEqual(expectedTasks); }); @@ -22,8 +28,18 @@ describe("extractTasks", () => { "- [x] completed task 1\n- [X] completed task 2" ); const expectedTasks = [ - { description: "completed task 1", checked: true }, - { description: "completed task 2", checked: true }, + { description: "completed task 1", checked: true, metadata: {}, + created: null, + due: null, + completion: null, + start: null, + scheduled: null, }, + { description: "completed task 2", checked: true, metadata: {}, + created: null, + due: null, + completion: null, + start: null, + scheduled: null, }, ]; expect(tasks).toEqual(expectedTasks); }); @@ -33,8 +49,18 @@ describe("extractTasks", () => { "- [x] completed task\n- [ ] uncompleted task" ); const expectedTasks = [ - { description: "completed task", checked: true }, - { description: "uncompleted task", checked: false }, + { description: "completed task", checked: true, metadata: {}, + created: null, + due: null, + completion: null, + start: null, + scheduled: null, }, + { description: "uncompleted task", checked: false, metadata: {}, + created: null, + due: null, + completion: null, + start: null, + scheduled: null, }, ]; expect(tasks).toEqual(expectedTasks); }); @@ -44,8 +70,18 @@ describe("extractTasks", () => { "- [x] completed task \n- [ ] uncompleted task " ); const expectedTasks = [ - { description: "completed task", checked: true }, - { description: "uncompleted task", checked: false }, + { description: "completed task", checked: true, metadata: {}, + created: null, + due: null, + completion: null, + start: null, + scheduled: null, }, + { description: "uncompleted task", checked: false, metadata: {}, + created: null, + due: null, + completion: null, + start: null, + scheduled: null, }, ]; expect(tasks).toEqual(expectedTasks); }); @@ -55,9 +91,24 @@ describe("extractTasks", () => { "- [x] task 1\n- [X] task 2\n- [ ] task 3" ); const expectedTasks = [ - { description: "task 1", checked: true }, - { description: "task 2", checked: true }, - { description: "task 3", checked: false }, + { description: "task 1", checked: true, metadata: {}, + created: null, + due: null, + completion: null, + start: null, + scheduled: null, }, + { description: "task 2", checked: true, metadata: {}, + created: null, + due: null, + completion: null, + start: null, + scheduled: null, }, + { description: "task 3", checked: false, metadata: {}, + created: null, + due: null, + completion: null, + start: null, + scheduled: null, }, ]; expect(tasks).toEqual(expectedTasks); }); @@ -65,7 +116,35 @@ describe("extractTasks", () => { test("should handle tasks with special characters", () => { const tasks = getTasksFromSource("- [x] task with $pecial character$"); const expectedTasks = [ - { description: "task with $pecial character$", checked: true }, + { + description: "task with $pecial character$", + checked: true, + metadata: {}, + created: null, + due: null, + completion: null, + start: null, + scheduled: null, + }, + ]; + expect(tasks).toEqual(expectedTasks); + }); + test("should handle tasks with metadata", () => { + const tasks = getTasksFromSource( + "- [x] task with metadata [field1:: field1value]" + ); + const expectedTasks = [ + { + description: "task with metadata [field1:: field1value]", + checked: true, + metadata: { field1 : "field1value", tags: [] }, + created: null, + due: null, + completion: null, + start: null, + scheduled: null, + + }, ]; expect(tasks).toEqual(expectedTasks); }); diff --git a/src/tests/parseFile.spec.ts b/src/tests/parseFile.spec.ts index 360a0bb..7447c94 100644 --- a/src/tests/parseFile.spec.ts +++ b/src/tests/parseFile.spec.ts @@ -51,8 +51,26 @@ describe("parseFile", () => { "Tag_avec_éèç-_öäüßñ", ], tasks: [ - { description: "uncompleted task", checked: false }, - { description: "completed task", checked: true }, + { + description: "uncompleted task", + checked: false, + metadata: {}, + created: null, + due: null, + completion: null, + start: null, + scheduled: null, + }, + { + description: "completed task", + checked: true, + metadata: {}, + created: null, + due: null, + completion: null, + start: null, + scheduled: null, + }, ], }; const expectedLinks = [ @@ -111,8 +129,26 @@ describe("parseFile", () => { "Tag_avec_éèç-_öäüßñ", ], tasks: [ - { description: "uncompleted task", checked: false }, - { description: "completed task", checked: true }, + { + description: "uncompleted task", + checked: false, + metadata: {}, + created: null, + due: null, + completion: null, + start: null, + scheduled: null, + }, + { + description: "completed task", + checked: true, + metadata: {}, + created: null, + due: null, + completion: null, + start: null, + scheduled: null, + }, ], }; const expectedLinks = [ @@ -160,3 +196,29 @@ describe("parseFile", () => { expect(links).toEqual(expectedLinks); }); }); +import { extractAllTaskMetadata } from "../lib/parseFile"; + +describe("extractAllTaskMetadata", () => { + it("should extract metadata fields from the description", () => { + const description = "[field1:: value1] [field2:: value2] [field3:: value3] #tag1 [due:: 2030-12-31] [created:: 2024-01-01 ] [completion:: ] [start:: ] [scheduled:: ] #tag2"; + const expectedMetadata = { + due: "2030-12-31", + field1: "value1", + field2: "value2", + field3: "value3", + created: "2024-01-01", + completion: "", + start: "", + scheduled: "", + tags: ["tag1", "tag2"], + }; + const metadata = extractAllTaskMetadata(description); + expect(metadata).toEqual(expectedMetadata); + }); + + it("should return an empty map if no metadata fields are found", () => { + const description = "This is a task description without any metadata"; + const metadata = extractAllTaskMetadata(description); + expect(metadata).toEqual({}); + }); +}); \ No newline at end of file diff --git a/src/tests/process.spec.ts b/src/tests/process.spec.ts index 719c5e3..4c9634c 100644 --- a/src/tests/process.spec.ts +++ b/src/tests/process.spec.ts @@ -46,14 +46,32 @@ describe("Can parse a file and get file info", () => { { checked: false, description: "uncompleted task 2", + metadata: {}, + created: null, + due: null, + completion: null, + start: null, + scheduled: null, }, { checked: true, description: "completed task 1", + metadata: {}, + created: null, + due: null, + completion: null, + start: null, + scheduled: null, }, { checked: true, description: "completed task 2", + metadata: {}, + created: null, + due: null, + completion: null, + start: null, + scheduled: null, }, ], });