From 393bfe4f4038cfff72df4420e03f0f3d738543d2 Mon Sep 17 00:00:00 2001 From: polsevev Date: Thu, 23 May 2024 13:46:40 +0200 Subject: [PATCH] optimized transformer and created evaluation --- output_files/pipeline_out.js | 4 + src/index.ts | 50 +++++++++- src/parser/parse.ts | 1 - src/test/test_transform.test.ts | 6 +- src/transform/transform.ts | 34 +++++-- src/transform/transformMatch.ts | 168 ++++++++++++++++++++++---------- 6 files changed, 195 insertions(+), 68 deletions(-) create mode 100644 output_files/pipeline_out.js diff --git a/output_files/pipeline_out.js b/output_files/pipeline_out.js new file mode 100644 index 0000000..696dfe6 --- /dev/null +++ b/output_files/pipeline_out.js @@ -0,0 +1,4 @@ +a |> w(%) |> w(%) |> w(%) |> w(%) |> w(%) |> w(%) |> w(%) |> w(%) |> w(%) |> w(%); +a |> b(%, a |> b(%, a |> b(%, a |> b(%, a |> b(%, a |> b(%, a |> b(%, b))))))); +a |> b(%, a) |> b(%, a) |> b(%, a) |> b(%, a) |> b(%, a) |> b(%, a) |> b(%, a) |> b(%, a) |> b(%, a) |> b(%, a) |> b(%, a) |> b(%, a); +b(b(b(b(a, a, a), a, a), a, a), a, a); \ No newline at end of file diff --git a/src/index.ts b/src/index.ts index 137a56e..5244e19 100644 --- a/src/index.ts +++ b/src/index.ts @@ -7,25 +7,65 @@ import { TransformRecipe, transform, } from "./transform/transform"; - +import { readdir } from "node:fs/promises"; import { parseJSTQL } from "./langium/langiumRunner"; const dir = "../prettier/src"; -const path = "test_files/test2.js"; +const path = "test_files/pipeline_test.js"; const file = Bun.file(path); const codeFromFile = await file.text(); const main = async () => { //transform(selfHostedTransformExampleMultiStmt, codeFromFile); + /* console.log(codeFromFile); const jstql_file = - "/home/rolfmg/Coding/Master/didactic-chainsaw/dsl_files/awaitToPromise.jstql"; + "/home/rolfmg/Coding/Master/didactic-chainsaw/dsl_files/pipeline.jstql"; const test_file = Bun.file(jstql_file); const test_JSTQL = await test_file.text(); let proposals = await parseJSTQL(test_JSTQL); - let code = transform(proposals[0].cases, codeFromFile); - await Bun.write("output_files/test2.js", code); + let [code, count] = transform(proposals[0].cases, codeFromFile); + await Bun.write("output_files/pipeline_out.js", code); + return; + */ + const jstql_file = + "/home/rolfmg/Coding/Master/didactic-chainsaw/dsl_files/pipeline.jstql"; + const test_file = Bun.file(jstql_file); + const test_JSTQL = await test_file.text(); + let proposals = await parseJSTQL(test_JSTQL); + + let basepathExamplesJSFiles = "../next.js"; + let examples = (await readdir(basepathExamplesJSFiles, { recursive: true })) + .filter((x) => x.endsWith(".js")) + .map((x) => basepathExamplesJSFiles + "/" + x); + console.log(examples); + + let sum = 0; + console.log("Scripts found ", sum, "matches!"); + let count = 0; + for (let examplesFile of examples) { + try { + if (examplesFile.split("/").includes("compiled")) { + continue; + } + console.log(examplesFile); + let script = await Bun.file(examplesFile).text(); + let [resultString, matches] = transform(proposals[0].cases, script); + sum += matches; + if (matches > 0) { + await Bun.write( + "output_testing/" + count + examplesFile.split("/").at(-1), + resultString + ); + count += 1; + } + } catch (e) { + console.log("Failed"); + } + console.log("current sum", sum); + } + console.log(sum); }; main(); diff --git a/src/parser/parse.ts b/src/parser/parse.ts index f27c48e..d82b1d1 100644 --- a/src/parser/parse.ts +++ b/src/parser/parse.ts @@ -74,7 +74,6 @@ export function parseInternalAplTo(code: string): InternalParseResult { cleanedJS += code[i]; } } - console.log(prelude, cleanedJS); return { prelude, cleanedJS }; } export interface Identifier extends WildcardNode { diff --git a/src/test/test_transform.test.ts b/src/test/test_transform.test.ts index ae01a26..bd4463e 100644 --- a/src/test/test_transform.test.ts +++ b/src/test/test_transform.test.ts @@ -22,13 +22,13 @@ let pipelineResFile = await Bun.file( "src/test/test_outputs/pipeline_output.js" ).text(); test("Test code: pipeline", () => { - expect(pipelineRes).toBe(pipelineResFile); + expect(pipelineRes).toEqual([pipelineResFile, 29]); }); let doRes = await runTest("test_files/do_test.js", "dsl_files/do.jstql"); let doResFile = await Bun.file("src/test/test_outputs/do_output.js").text(); test("Test code: do", () => { - expect(doRes).toBe(doResFile); + expect(doRes).toEqual([doResFile, 2]); }); let awaitToPromise = await runTest( @@ -40,5 +40,5 @@ let awaitToPromiseOutput = await Bun.file( "src/test/test_outputs/awaitToPromise_output.js" ).text(); test("Test code: await to promise", () => { - expect(awaitToPromise).toBe(awaitToPromiseOutput); + expect(awaitToPromise).toEqual([awaitToPromiseOutput, 1]); }); diff --git a/src/transform/transform.ts b/src/transform/transform.ts index 6833ca0..6702764 100644 --- a/src/transform/transform.ts +++ b/src/transform/transform.ts @@ -28,13 +28,16 @@ export interface TransformRecipe { export interface SelfHostedRecipe extends TransformRecipe { prelude: string; } -export function transform(recipes: TransformRecipe[], code: string): string { +export function transform( + recipes: TransformRecipe[], + code: string +): [string, number] { let codeAST: t.Node = parse_with_plugins(code); - + let amount = 0; for (let recipe of recipes) { if ((recipe).prelude !== undefined) { // We are using the self hosted version - codeAST = transformSelfHosted( + let temp = transformSelfHosted( { applicableTo: recipe.applicableTo, transformTo: recipe.transformTo, @@ -42,6 +45,8 @@ export function transform(recipes: TransformRecipe[], code: string): string { preludeBuilder((recipe as SelfHostedRecipe).prelude), codeAST ); + codeAST = temp[0]; + amount += temp[1]; } else { // We are using JSTQL // We have to parse JSTQL to the self hosted version @@ -51,24 +56,26 @@ export function transform(recipes: TransformRecipe[], code: string): string { ); let transformTo = parseInternalTraTo(recipe.transformTo); - codeAST = transformSelfHosted( + let temp = transformSelfHosted( { applicableTo, transformTo }, prelude, codeAST ); + codeAST = temp[0]; + amount += temp[1]; } } let output = generate(codeAST, { topicToken: "%" }).code; //showTree(transformToTree); - return output; + return [output, amount]; } function transformSelfHosted( recipe: TransformRecipe, internals: Wildcard[], codeAST: t.Node -): t.Node { +): [t.Node, number] { let codeTree = makeTree(codeAST as babelparser.ParseResult); let applicabelToAST = parse_with_plugins(recipe.applicableTo); @@ -83,9 +90,18 @@ function transformSelfHosted( ) { throw new Error("This no worky LOL"); } - let matches = runMatch(codeTree, applicableToTree, internals); + console.log("We found", matches.length, "matches"); - let outputAST = transformer(matches, transformToTree, codeAST, transformTo); - return outputAST; + + let outputAST = transformer( + matches, + transformToTree, + codeAST, + transformTo, + internals.map((x) => x.identifier.name) + ); + + console.log("Finished transforming"); + return [outputAST, matches.length]; } diff --git a/src/transform/transformMatch.ts b/src/transform/transformMatch.ts index 4658c9a..df88a48 100644 --- a/src/transform/transformMatch.ts +++ b/src/transform/transformMatch.ts @@ -12,78 +12,146 @@ import { Match, MatchedTreeNode, PairedNodes } from "../matcher/matcher"; import traverse from "@babel/traverse"; import generate from "@babel/generator"; import { TransformRecipe } from "./transform"; +import { Wildcard } from "../parser/parse"; export function transformer( matches: Match[], transformTo: TreeNode, codeAST: t.Node, - traToAST: t.File + traToAST: t.File, + wildcardIdentifiers: string[] ): t.Node { + let transformedTransformTo: [Match, t.File][] = []; for (let match of matches.reverse()) { try { let traToWithWildcards = structuredClone(traToAST); for (let match_stmt of match.statements) { + let aplToWildcards: Map = new Map(); + FindWildcardsAplTo( + match_stmt, + wildcardIdentifiers, + aplToWildcards + ); transformMatch(match_stmt, transformTo, traToWithWildcards); } - traverse(codeAST, { - enter(path) { - if ( - !( - path.node.type === "Program" || - path.node.type === "File" - ) - ) { - if ( - path.node === - match.statements[0].element.codeNode[0] - ) { - path.replaceWithMultiple( - traToWithWildcards.program.body - ); - let siblings = path.getAllNextSiblings(); - // For multi line applicable to - for ( - let i = 0; - i < match.statements.length - 1; - i++ - ) { - siblings[i].remove(); - } + transformedTransformTo.push([match, traToWithWildcards]); + } catch (e) { + console.log(e); + } + //console.log("Finished replacement"); + } + for (let [match, traToWithWildcards] of transformedTransformTo) { + traverse(codeAST, { + enter(path) { + if ( + !(path.node.type === "Program" || path.node.type === "File") + ) { + if (path.node === match.statements[0].element.codeNode[0]) { + path.replaceWithMultiple( + traToWithWildcards.program.body + ); + let siblings = path.getAllNextSiblings(); - // For when we have matched with * + // For multi line applicable to + for (let i = 0; i < match.statements.length - 1; i++) { + siblings[i].remove(); + } - for (let matchStmt of match.statements) { - for (let [ - i, - stmtMatchedWithStar, - ] of matchStmt.element.codeNode.entries()) { - let siblingnodes = siblings.map( - (a) => a.node - ); - if ( - siblingnodes.includes( + // For when we have matched with + + + for (let matchStmt of match.statements) { + for (let [ + i, + stmtMatchedWithStar, + ] of matchStmt.element.codeNode.entries()) { + let siblingnodes = siblings.map((a) => a.node); + if ( + siblingnodes.includes(stmtMatchedWithStar) + ) { + let index = + siblingnodes.indexOf( stmtMatchedWithStar - ) - ) { - let index = - siblingnodes.indexOf( - stmtMatchedWithStar - ); - siblings[index].remove(); - } + ); + siblings[index].remove(); } } } } - }, - }); - } catch (e) { - console.log(e); - } + } + }, + }); + } + return codeAST; +} + +function FindWildcardsAplTo( + match: TreeNode, + wildcardIdentifiers: string[], + out: Map +) { + let ident = isWildcardIdent(wildcardIdentifiers, match.element.aplToNode); + if (!ident) { + return; } - return codeAST; + out.set(ident, match.element.codeNode); + + for (let child of match.children) { + FindWildcardsAplTo(child, wildcardIdentifiers, out); + } +} + +function isWildcardIdent( + wildcards: string[], + aplToNode: t.Node +): string | undefined { + if (aplToNode.type === "Identifier") { + for (let ident of wildcards) { + if (aplToNode.name === ident) { + return ident; + } + } + } else if ( + aplToNode.type === "ExpressionStatement" && + aplToNode.expression.type === "Identifier" + ) { + for (let ident of wildcards) { + if (aplToNode.expression.name === ident) { + return ident; + } + } + } + return undefined; +} + +export function transformTransformTo( + wildcardsMatchedWith: Map, + output: t.Node +) { + traverse(output, { + Identifier: (path) => { + if (wildcardsMatchedWith.has(path.node.name)) { + let newNode = wildcardsMatchedWith.get(path.node.name); + if (newNode) { + path.replaceWithMultiple(newNode); + } + } + }, + ExpressionStatement: (path) => { + if ( + path.node.expression.type === "Identifier" && + wildcardsMatchedWith.has(path.node.expression.name) + ) { + let newNode = wildcardsMatchedWith.get( + path.node.expression.name + ); + if (newNode) { + path.replaceWithMultiple(newNode); + } + } + }, + }); } export function transformMatch(