optimized transformer and created evaluation

output_files/pipeline_out.js

@@ -0,0 +1,4 @@
+a |> w(%) |> w(%) |> w(%) |> w(%) |> w(%) |> w(%) |> w(%) |> w(%) |> w(%) |> w(%);
+a |> b(%, a |> b(%, a |> b(%, a |> b(%, a |> b(%, a |> b(%, a |> b(%, b)))))));
+a |> b(%, a) |> b(%, a) |> b(%, a) |> b(%, a) |> b(%, a) |> b(%, a) |> b(%, a) |> b(%, a) |> b(%, a) |> b(%, a) |> b(%, a) |> b(%, a);
+b(b(b(b(a, a, a), a, a), a, a), a, a);

src/index.ts

@@ -7,25 +7,65 @@ import {
     TransformRecipe,
     transform,
 } from "./transform/transform";
-
+import { readdir } from "node:fs/promises";
 import { parseJSTQL } from "./langium/langiumRunner";
 
 const dir = "../prettier/src";
 
-const path = "test_files/test2.js";
+const path = "test_files/pipeline_test.js";
 const file = Bun.file(path);
 const codeFromFile = await file.text();
 const main = async () => {
     //transform(selfHostedTransformExampleMultiStmt, codeFromFile);
+    /*
     console.log(codeFromFile);
     const jstql_file =
-        "/home/rolfmg/Coding/Master/didactic-chainsaw/dsl_files/awaitToPromise.jstql";
+        "/home/rolfmg/Coding/Master/didactic-chainsaw/dsl_files/pipeline.jstql";
     const test_file = Bun.file(jstql_file);
     const test_JSTQL = await test_file.text();
     let proposals = await parseJSTQL(test_JSTQL);
 
-    let code = transform(proposals[0].cases, codeFromFile);
+    let [code, count] = transform(proposals[0].cases, codeFromFile);
-    await Bun.write("output_files/test2.js", code);
+    await Bun.write("output_files/pipeline_out.js", code);
+    return;
+    */
+    const jstql_file =
+        "/home/rolfmg/Coding/Master/didactic-chainsaw/dsl_files/pipeline.jstql";
+    const test_file = Bun.file(jstql_file);
+    const test_JSTQL = await test_file.text();
+    let proposals = await parseJSTQL(test_JSTQL);
+
+    let basepathExamplesJSFiles = "../next.js";
+    let examples = (await readdir(basepathExamplesJSFiles, { recursive: true }))
+        .filter((x) => x.endsWith(".js"))
+        .map((x) => basepathExamplesJSFiles + "/" + x);
+    console.log(examples);
+
+    let sum = 0;
+    console.log("Scripts found ", sum, "matches!");
+    let count = 0;
+    for (let examplesFile of examples) {
+        try {
+            if (examplesFile.split("/").includes("compiled")) {
+                continue;
+            }
+            console.log(examplesFile);
+            let script = await Bun.file(examplesFile).text();
+            let [resultString, matches] = transform(proposals[0].cases, script);
+            sum += matches;
+            if (matches > 0) {
+                await Bun.write(
+                    "output_testing/" + count + examplesFile.split("/").at(-1),
+                    resultString
+                );
+                count += 1;
+            }
+        } catch (e) {
+            console.log("Failed");
+        }
+        console.log("current sum", sum);
+    }
+    console.log(sum);
 };
 
 main();

src/parser/parse.ts

@@ -74,7 +74,6 @@ export function parseInternalAplTo(code: string): InternalParseResult {
             cleanedJS += code[i];
         }
     }
-    console.log(prelude, cleanedJS);
     return { prelude, cleanedJS };
 }
 export interface Identifier extends WildcardNode {

src/test/test_transform.test.ts

@@ -22,13 +22,13 @@ let pipelineResFile = await Bun.file(
     "src/test/test_outputs/pipeline_output.js"
 ).text();
 test("Test code: pipeline", () => {
-    expect(pipelineRes).toBe(pipelineResFile);
+    expect(pipelineRes).toEqual([pipelineResFile, 29]);
 });
 let doRes = await runTest("test_files/do_test.js", "dsl_files/do.jstql");
 
 let doResFile = await Bun.file("src/test/test_outputs/do_output.js").text();
 test("Test code: do", () => {
-    expect(doRes).toBe(doResFile);
+    expect(doRes).toEqual([doResFile, 2]);
 });
 
 let awaitToPromise = await runTest(
@@ -40,5 +40,5 @@ let awaitToPromiseOutput = await Bun.file(
     "src/test/test_outputs/awaitToPromise_output.js"
 ).text();
 test("Test code: await to promise", () => {
-    expect(awaitToPromise).toBe(awaitToPromiseOutput);
+    expect(awaitToPromise).toEqual([awaitToPromiseOutput, 1]);
 });

src/transform/transform.ts

@@ -28,13 +28,16 @@ export interface TransformRecipe {
 export interface SelfHostedRecipe extends TransformRecipe {
     prelude: string;
 }
-export function transform(recipes: TransformRecipe[], code: string): string {
+export function transform(
+    recipes: TransformRecipe[],
+    code: string
+): [string, number] {
     let codeAST: t.Node = parse_with_plugins(code);
-
+    let amount = 0;
     for (let recipe of recipes) {
         if ((<SelfHostedRecipe>recipe).prelude !== undefined) {
             // We are using the self hosted version
-            codeAST = transformSelfHosted(
+            let temp = transformSelfHosted(
                 {
                     applicableTo: recipe.applicableTo,
                     transformTo: recipe.transformTo,
@@ -42,6 +45,8 @@ export function transform(recipes: TransformRecipe[], code: string): string {
                 preludeBuilder((recipe as SelfHostedRecipe).prelude),
                 codeAST
             );
+            codeAST = temp[0];
+            amount += temp[1];
         } else {
             // We are using JSTQL
             // We have to parse JSTQL to the self hosted version
@@ -51,24 +56,26 @@ export function transform(recipes: TransformRecipe[], code: string): string {
             );
             let transformTo = parseInternalTraTo(recipe.transformTo);
 
-            codeAST = transformSelfHosted(
+            let temp = transformSelfHosted(
                 { applicableTo, transformTo },
                 prelude,
                 codeAST
             );
+            codeAST = temp[0];
+            amount += temp[1];
         }
     }
 
     let output = generate(codeAST, { topicToken: "%" }).code;
     //showTree(transformToTree);
-    return output;
+    return [output, amount];
 }
 
 function transformSelfHosted(
     recipe: TransformRecipe,
     internals: Wildcard[],
     codeAST: t.Node
-): t.Node {
+): [t.Node, number] {
     let codeTree = makeTree(codeAST as babelparser.ParseResult<t.File>);
     let applicabelToAST = parse_with_plugins(recipe.applicableTo);
 
@@ -83,9 +90,18 @@ function transformSelfHosted(
     ) {
         throw new Error("This no worky LOL");
     }
-
     let matches = runMatch(codeTree, applicableToTree, internals);
+
     console.log("We found", matches.length, "matches");
-    let outputAST = transformer(matches, transformToTree, codeAST, transformTo);
+
-    return outputAST;
+    let outputAST = transformer(
+        matches,
+        transformToTree,
+        codeAST,
+        transformTo,
+        internals.map((x) => x.identifier.name)
+    );
+
+    console.log("Finished transforming");
+    return [outputAST, matches.length];
 }

src/transform/transformMatch.ts

@@ -12,59 +12,62 @@ import { Match, MatchedTreeNode, PairedNodes } from "../matcher/matcher";
 import traverse from "@babel/traverse";
 import generate from "@babel/generator";
 import { TransformRecipe } from "./transform";
+import { Wildcard } from "../parser/parse";
 
 export function transformer(
     matches: Match[],
     transformTo: TreeNode<t.Node>,
     codeAST: t.Node,
-    traToAST: t.File
+    traToAST: t.File,
+    wildcardIdentifiers: string[]
 ): t.Node {
+    let transformedTransformTo: [Match, t.File][] = [];
     for (let match of matches.reverse()) {
         try {
             let traToWithWildcards = structuredClone(traToAST);
             for (let match_stmt of match.statements) {
+                let aplToWildcards: Map<string, t.Node[]> = new Map();
+                FindWildcardsAplTo(
+                    match_stmt,
+                    wildcardIdentifiers,
+                    aplToWildcards
+                );
                 transformMatch(match_stmt, transformTo, traToWithWildcards);
             }
+
+            transformedTransformTo.push([match, traToWithWildcards]);
+        } catch (e) {
+            console.log(e);
+        }
+        //console.log("Finished replacement");
+    }
+    for (let [match, traToWithWildcards] of transformedTransformTo) {
         traverse(codeAST, {
             enter(path) {
                 if (
-                        !(
+                    !(path.node.type === "Program" || path.node.type === "File")
-                            path.node.type === "Program" ||
-                            path.node.type === "File"
-                        )
-                    ) {
-                        if (
-                            path.node ===
-                            match.statements[0].element.codeNode[0]
                 ) {
+                    if (path.node === match.statements[0].element.codeNode[0]) {
                         path.replaceWithMultiple(
                             traToWithWildcards.program.body
                         );
                         let siblings = path.getAllNextSiblings();
 
                         // For multi line applicable to
-                            for (
+                        for (let i = 0; i < match.statements.length - 1; i++) {
-                                let i = 0;
-                                i < match.statements.length - 1;
-                                i++
-                            ) {
                             siblings[i].remove();
                         }
 
-                            // For when we have matched with *
+                        // For when we have matched with +
 
                         for (let matchStmt of match.statements) {
                             for (let [
                                 i,
                                 stmtMatchedWithStar,
                             ] of matchStmt.element.codeNode.entries()) {
-                                    let siblingnodes = siblings.map(
+                                let siblingnodes = siblings.map((a) => a.node);
-                                        (a) => a.node
-                                    );
                                 if (
-                                        siblingnodes.includes(
+                                    siblingnodes.includes(stmtMatchedWithStar)
-                                            stmtMatchedWithStar
-                                        )
                                 ) {
                                     let index =
                                         siblingnodes.indexOf(
@@ -78,12 +81,77 @@ export function transformer(
                 }
             },
         });
-        } catch (e) {
+    }
-            console.log(e);
+    return codeAST;
+}
+
+function FindWildcardsAplTo(
+    match: TreeNode<PairedNodes>,
+    wildcardIdentifiers: string[],
+    out: Map<string, t.Node[]>
+) {
+    let ident = isWildcardIdent(wildcardIdentifiers, match.element.aplToNode);
+    if (!ident) {
+        return;
+    }
+
+    out.set(ident, match.element.codeNode);
+
+    for (let child of match.children) {
+        FindWildcardsAplTo(child, wildcardIdentifiers, out);
     }
 }
 
-    return codeAST;
+function isWildcardIdent(
+    wildcards: string[],
+    aplToNode: t.Node
+): string | undefined {
+    if (aplToNode.type === "Identifier") {
+        for (let ident of wildcards) {
+            if (aplToNode.name === ident) {
+                return ident;
+            }
+        }
+    } else if (
+        aplToNode.type === "ExpressionStatement" &&
+        aplToNode.expression.type === "Identifier"
+    ) {
+        for (let ident of wildcards) {
+            if (aplToNode.expression.name === ident) {
+                return ident;
+            }
+        }
+    }
+    return undefined;
+}
+
+export function transformTransformTo(
+    wildcardsMatchedWith: Map<string, t.Node>,
+    output: t.Node
+) {
+    traverse(output, {
+        Identifier: (path) => {
+            if (wildcardsMatchedWith.has(path.node.name)) {
+                let newNode = wildcardsMatchedWith.get(path.node.name);
+                if (newNode) {
+                    path.replaceWithMultiple(newNode);
+                }
+            }
+        },
+        ExpressionStatement: (path) => {
+            if (
+                path.node.expression.type === "Identifier" &&
+                wildcardsMatchedWith.has(path.node.expression.name)
+            ) {
+                let newNode = wildcardsMatchedWith.get(
+                    path.node.expression.name
+                );
+                if (newNode) {
+                    path.replaceWithMultiple(newNode);
+                }
+            }
+        },
+    });
 }
 
 export function transformMatch(