Basically finito, needs some final polishing and writing a README

JSTQL/src/language/jstql-validator.ts

@@ -1,5 +1,5 @@
 import type { ValidationAcceptor, ValidationChecks } from "langium";
-import type { JstqlAstType, Pair } from "./generated/ast.js";
+import type { JstqlAstType, Case } from "./generated/ast.js";
 import type { JstqlServices } from "./jstql-module.js";
 
 /**
@@ -9,7 +9,7 @@ export function registerValidationChecks(services: JstqlServices) {
     const registry = services.validation.ValidationRegistry;
     const validator = services.validation.JstqlValidator;
     const checks: ValidationChecks<JstqlAstType> = {
-        Pair: validator.validateWildcards,
+        Case: validator.validateWildcards,
     };
     registry.register(checks, validator);
 }
@@ -18,7 +18,7 @@ export function registerValidationChecks(services: JstqlServices) {
  * Implementation of custom validations.
  */
 export class JstqlValidator {
-    validateWildcards(pair: Pair, accept: ValidationAcceptor): void {
+    validateWildcards(pair: Case, accept: ValidationAcceptor): void {
         try {
             let validationResultAplTo = validateWildcardAplTo(
                 collectWildcard(pair.aplTo.apl_to_code.split(""))

JSTQL/src/language/jstql.langium

@@ -5,11 +5,11 @@ entry Model:
 
 Proposal:
     'proposal' name=ID "{"
-        (pair+=Pair)+
+        (case+=Case)+
     "}";
 
-Pair:
+Case:
-    "pair" name=ID "{"
+    "case" name=ID "{"
         aplTo=ApplicableTo
         traTo=TraTo
     "}";

dsl_files/awaitToPromise.jstql

@@ -0,0 +1,17 @@
+proposal awaitToPomise{
+    case single{
+        applicable to {
+            "let <<ident:Identifier>> = await <<awaitedExpr: Expression>>;
+            <<statements: (Statement && !ReturnStatement)*>>
+            return <<returnExpr: Expression>>
+            "
+        }
+
+        transform to{
+            "return <<awaitedExpr>>.then((<<ident>>) => {
+                <<statements>>
+                <<returnExpr>>
+            });"
+        }
+    }
+}

dsl_files/do.jstql

@@ -1,9 +1,9 @@
 proposal DoExpression{
-    pair arrowFunction{
+    case arrowFunction{
         applicable to {
             "let <<ident:Identifier>> = () => {
-                <<statements: Statement && !ReturnStatement *>>
+                <<statements: (Statement && !ReturnStatement)*>>
-                return <<returnVal : Expression | Identifier>>;
+                return <<returnVal : Expression>>;
             }
             "
         }
@@ -15,11 +15,11 @@ proposal DoExpression{
         }
     }
 
-    pair immediatelyInvokedUnnamedFunction {
+    case immediatelyInvokedUnnamedFunction {
         applicable to {
             "let <<ident:Identifier>> = function(){
-                <<statements: Statement && !ReturnStatement >>
+                <<statements: (Statement && !ReturnStatement)*>>
-                return <<returnVal : Expression | Identifier>>;
+                return <<returnVal : Expression>>;
             }();"
         }
 

dsl_files/multi_stmt_test.jstql

@@ -1,5 +1,5 @@
 proposal MultiStmt{
-    pair Smthn{
+    case Smthn{
         applicable to{
             "let <<ident1:Identifier>> = <<funcIdent:Identifier | MemberExpression>>();
             let <<ident2:Identifier>> = <<expr:Expression>>;

dsl_files/pipeline.jstql

@@ -1,5 +1,6 @@
 proposal Pipeline{
-    pair SingleArgument {
+
+    case SingleArgument {
         applicable to {
             "<<someFunctionIdent:Identifier || MemberExpression>>(<<someFunctionParam: Expression>>);"
         }
@@ -8,4 +9,13 @@ proposal Pipeline{
             "<<someFunctionParam>> |> <<someFunctionIdent>>(%);"
         }
     }
+
+    case TwoArgument{
+        applicable to {
+            "<<someFunctionIdent: Identifier || MemberExpression>>(<<someFunctionParam: Expression>>, <<moreFunctionParam: Expression>>)"
+        }
+        transform to {
+            "<<someFunctionParam>> |> <<someFunctionIdent>>(%, <<moreFunctionParam>>)"
+        }
+    }
 }

dsl_files/star.jstql

@@ -0,0 +1,17 @@
+proposal Star{
+    case a {
+        applicable to {
+            "let <<ident:Identifier>> = () => {
+                <<statements: Statement>>
+                return <<returnVal : Expression>>;
+            }
+            "
+        }
+        transform to {
+            "let <<ident>> = do {
+                <<statements>>
+                <<returnVal>>
+            }"
+        }
+    }
+}

dsl_files/test.jstl

@@ -1,14 +0,0 @@
-
-
-
-proposal async {
-    applicable to {
-        let _$_a_$_ = await _$_expr_$_();
-        console.log(_$_a_$_);
-    }
-    replace with {
-        _$_expr_$_().then(() => {
-            console.log(_$_a_$_);
-        })
-    }
-}

dsl_files/test_hello.jstl

@@ -1,11 +0,0 @@
-proposal p1 {
-    pair something{
-        applicable to {
-            "let a = 0;"
-        }
-        transform to {
-            ""
-        }
-    }
-}
-

output_files/output_await_to_promise.js

@@ -0,0 +1,100 @@
+// "fast-glob" and `createTwoFilesPatch` are bundled here since the API uses `micromatch` and `diff` too
+import { createTwoFilesPatch } from "diff/lib/patch/create.js";
+import fastGlob from "fast-glob";
+import * as vnopts from "vnopts";
+import * as errors from "./common/errors.js";
+import getFileInfoWithoutPlugins from "./common/get-file-info.js";
+import mockable from "./common/mockable.js";
+import { clearCache as clearConfigCache, resolveConfig, resolveConfigFile } from "./config/resolve-config.js";
+import * as core from "./main/core.js";
+import { formatOptionsHiddenDefaults } from "./main/normalize-format-options.js";
+import normalizeOptions from "./main/normalize-options.js";
+import * as optionCategories from "./main/option-categories.js";
+import { clearCache as clearPluginCache, loadBuiltinPlugins, loadPlugins } from "./main/plugins/index.js";
+import { getSupportInfo as getSupportInfoWithoutPlugins, normalizeOptionSettings } from "./main/support.js";
+import { createIsIgnoredFunction } from "./utils/ignore.js";
+import isNonEmptyArray from "./utils/is-non-empty-array.js";
+import omit from "./utils/object-omit.js";
+import partition from "./utils/partition.js";
+
+/**
+ * @param {*} fn
+ * @param {number} [optionsArgumentIndex]
+ * @returns {*}
+ */
+function withPlugins(fn, optionsArgumentIndex = 1 // Usually `options` is the 2nd argument
+) {
+  return async (...args) => {
+    const options = args[optionsArgumentIndex] ?? {};
+    const {
+      plugins = []
+    } = options;
+    args[optionsArgumentIndex] = {
+      ...options,
+      plugins: (await Promise.all([loadBuiltinPlugins(),
+      // TODO: standalone version allow `plugins` to be `prettierPlugins` which is an object, should allow that too
+      loadPlugins(plugins)])).flat()
+    };
+    return fn(...args);
+  };
+}
+const formatWithCursor = withPlugins(core.formatWithCursor);
+async function format(text, options) {
+  const {
+    formatted
+  } = await formatWithCursor(text, {
+    ...options,
+    cursorOffset: -1
+  });
+  return formatted;
+}
+async function check(text, options) {
+  return (await format(text, options)) === text;
+}
+
+// eslint-disable-next-line require-await
+async function clearCache() {
+  clearConfigCache();
+  clearPluginCache();
+}
+
+/** @type {typeof getFileInfoWithoutPlugins} */
+const getFileInfo = withPlugins(getFileInfoWithoutPlugins);
+
+/** @type {typeof getSupportInfoWithoutPlugins} */
+const getSupportInfo = withPlugins(getSupportInfoWithoutPlugins, 0);
+
+// Internal shared with cli
+const sharedWithCli = {
+  errors,
+  optionCategories,
+  createIsIgnoredFunction,
+  formatOptionsHiddenDefaults,
+  normalizeOptions,
+  getSupportInfoWithoutPlugins,
+  normalizeOptionSettings,
+  vnopts: {
+    ChoiceSchema: vnopts.ChoiceSchema,
+    apiDescriptor: vnopts.apiDescriptor
+  },
+  fastGlob,
+  createTwoFilesPatch,
+  utils: {
+    isNonEmptyArray,
+    partition,
+    omit
+  },
+  mockable
+};
+const debugApis = {
+  parse: withPlugins(core.parse),
+  formatAST: withPlugins(core.formatAst),
+  formatDoc: withPlugins(core.formatDoc),
+  printToDoc: withPlugins(core.printToDoc),
+  printDocToString: withPlugins(core.printDocToString),
+  mockable
+};
+export { debugApis as __debug, sharedWithCli as __internal, check, clearCache as clearConfigCache, format, formatWithCursor, getFileInfo, getSupportInfo, resolveConfig, resolveConfigFile };
+export * as doc from "./document/public.js";
+export { default as version } from "./main/version.evaluate.cjs";
+export * as util from "./utils/public.js";

output_files/output_do.js

@@ -0,0 +1,12 @@
+let aaaa = do {
+  let g = 100;
+  let ff = 10;
+  let ggg = a(b);
+  100;
+};
+let bbaaa = do {
+  let lllll = 1 + 1;
+  100 + 100;
+  const aaaaa = aaaa(bb);
+  lllll;
+};

output_files/output_pipeline.js

@@ -0,0 +1,4 @@
+a |> w(%) |> w(%) |> w(%) |> w(%) |> w(%) |> w(%) |> w(%) |> w(%) |> w(%) |> w(%);
+a |> b(%, a |> b(%, a |> b(%, a |> b(%, a |> b(%, a |> b(%, a |> b(%, b)))))));
+a |> b(%, a) |> b(%, a) |> b(%, a) |> b(%, a) |> b(%, a) |> b(%, a) |> b(%, a) |> b(%, a) |> b(%, a) |> b(%, a) |> b(%, a) |> b(%, a);
+b(b(b(b(a, a, a), a, a), a, a), a, a);

output_files/testingLOL.js

@@ -0,0 +1,98 @@
+// "fast-glob" and `createTwoFilesPatch` are bundled here since the API uses `micromatch` and `diff` too
+import { createTwoFilesPatch } from "diff/lib/patch/create.js";
+import fastGlob from "fast-glob";
+import * as vnopts from "vnopts";
+import * as errors from "./common/errors.js";
+import getFileInfoWithoutPlugins from "./common/get-file-info.js";
+import mockable from "./common/mockable.js";
+import { clearCache as clearConfigCache, resolveConfig, resolveConfigFile } from "./config/resolve-config.js";
+import * as core from "./main/core.js";
+import { formatOptionsHiddenDefaults } from "./main/normalize-format-options.js";
+import normalizeOptions from "./main/normalize-options.js";
+import * as optionCategories from "./main/option-categories.js";
+import { clearCache as clearPluginCache, loadBuiltinPlugins, loadPlugins } from "./main/plugins/index.js";
+import { getSupportInfo as getSupportInfoWithoutPlugins, normalizeOptionSettings } from "./main/support.js";
+import { createIsIgnoredFunction } from "./utils/ignore.js";
+import isNonEmptyArray from "./utils/is-non-empty-array.js";
+import omit from "./utils/object-omit.js";
+import partition from "./utils/partition.js";
+
+/**
+ * @param {*} fn
+ * @param {number} [optionsArgumentIndex]
+ * @returns {*}
+ */
+function withPlugins(fn, optionsArgumentIndex = 1 // Usually `options` is the 2nd argument
+) {
+  return async (...args) => {
+    const options = args[optionsArgumentIndex] ?? {};
+    const {
+      plugins = []
+    } = options;
+    args[optionsArgumentIndex] = {
+      ...options,
+      plugins: (await ([loadBuiltinPlugins(), plugins |> loadPlugins(%)] |> Promise.all(%))).flat()
+    };
+    return fn(...args);
+  };
+}
+const formatWithCursor = core.formatWithCursor |> withPlugins(%);
+async function format(text, options) {
+  const {
+    formatted
+  } = await (text |> formatWithCursor(%, {
+    ...options,
+    cursorOffset: -1
+  }));
+  return formatted;
+}
+async function check(text, options) {
+  return (await (text |> format(%, options))) === text;
+}
+
+// eslint-disable-next-line require-await
+async function clearCache() {
+  clearConfigCache();
+  clearPluginCache();
+}
+
+/** @type {typeof getFileInfoWithoutPlugins} */
+const getFileInfo = getFileInfoWithoutPlugins |> withPlugins(%);
+
+/** @type {typeof getSupportInfoWithoutPlugins} */
+const getSupportInfo = getSupportInfoWithoutPlugins |> withPlugins(%, 0);
+
+// Internal shared with cli
+const sharedWithCli = {
+  errors,
+  optionCategories,
+  createIsIgnoredFunction,
+  formatOptionsHiddenDefaults,
+  normalizeOptions,
+  getSupportInfoWithoutPlugins,
+  normalizeOptionSettings,
+  vnopts: {
+    ChoiceSchema: vnopts.ChoiceSchema,
+    apiDescriptor: vnopts.apiDescriptor
+  },
+  fastGlob,
+  createTwoFilesPatch,
+  utils: {
+    isNonEmptyArray,
+    partition,
+    omit
+  },
+  mockable
+};
+const debugApis = {
+  parse: core.parse |> withPlugins(%),
+  formatAST: core.formatAst |> withPlugins(%),
+  formatDoc: core.formatDoc |> withPlugins(%),
+  printToDoc: core.printToDoc |> withPlugins(%),
+  printDocToString: core.printDocToString |> withPlugins(%),
+  mockable
+};
+export { debugApis as __debug, sharedWithCli as __internal, check, clearCache as clearConfigCache, format, formatWithCursor, getFileInfo, getSupportInfo, resolveConfig, resolveConfigFile };
+export * as doc from "./document/public.js";
+export { default as version } from "./main/version.evaluate.cjs";
+export * as util from "./utils/public.js";

package-lock.json

@@ -19,6 +19,7 @@
       },
       "devDependencies": {
         "@babel/plugin-proposal-pipeline-operator": "^7.23.3",
+        "@babel/plugin-syntax-top-level-await": "^7.14.5",
         "@swc/cli": "^0.1.62",
         "@types/babel__generator": "^7.6.8",
         "@types/node": "^20.5.9",
@@ -337,6 +338,21 @@
         "@babel/core": "^7.0.0-0"
       }
     },
+    "node_modules/@babel/plugin-syntax-top-level-await": {
+      "version": "7.14.5",
+      "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-top-level-await/-/plugin-syntax-top-level-await-7.14.5.tgz",
+      "integrity": "sha512-hx++upLv5U1rgYfwe1xBQUhRmU41NEvpUvrp8jkrSCdvGSnM5/qdRMtylJ6PG5OFkBaHkbTAKTnd3/YyESRHFw==",
+      "dev": true,
+      "dependencies": {
+        "@babel/helper-plugin-utils": "^7.14.5"
+      },
+      "engines": {
+        "node": ">=6.9.0"
+      },
+      "peerDependencies": {
+        "@babel/core": "^7.0.0-0"
+      }
+    },
     "node_modules/@babel/template": {
       "version": "7.22.15",
       "resolved": "https://registry.npmjs.org/@babel/template/-/template-7.22.15.tgz",

package.json

@@ -8,6 +8,7 @@
   },
   "devDependencies": {
     "@babel/plugin-proposal-pipeline-operator": "^7.23.3",
+    "@babel/plugin-syntax-top-level-await": "^7.14.5",
     "@swc/cli": "^0.1.62",
     "@types/babel__generator": "^7.6.8",
     "@types/node": "^20.5.9",

src/data_structures/tree.ts

@@ -57,7 +57,7 @@ export const showTreePaired = (
     console.log(
         "   ".repeat(idents),
         tree.element.aplToNode.type,
-        tree.element.codeNode.type
+        tree.element.codeNode.map((x) => x.type)
     );
     tree.children.forEach((child) => {
         showTreePaired(child, idents + 1);

src/index.ts

@@ -9,22 +9,23 @@ import {
 } from "./transform/transform";
 
 import { parseJSTQL } from "./langium/langiumRunner";
-const path = "test_files/test.js";
+
+const dir = "../prettier/src";
+
+const path = "../prettier/src/index.js";
 const file = Bun.file(path);
 const codeFromFile = await file.text();
 const main = async () => {
     //transform(selfHostedTransformExampleMultiStmt, codeFromFile);
-
+    console.log(codeFromFile);
     const jstql_file =
         "/home/rolfmg/Coding/Master/didactic-chainsaw/dsl_files/pipeline.jstql";
     const test_file = Bun.file(jstql_file);
     const test_JSTQL = await test_file.text();
     let proposals = await parseJSTQL(test_JSTQL);
 
-    await Bun.write(
+    let code = transform(proposals[0].cases, codeFromFile);
-        "output_files/output.js",
+    await Bun.write("output_files/testingLOL.js", code);
-        transform(proposals[0].pairs[0], codeFromFile)
-    );
 };
 
 main();

src/langium/langiumRunner.ts

@@ -1,19 +1,20 @@
 import { TransformRecipe, Proposal as LocalProp } from "../transform/transform";
 import { parseDSLtoAST } from "../../JSTQL/src/JSTQL_interface/api";
-import { Model, Proposal } from "../../JSTQL/src/language/generated/ast";
+import { Model, Case } from "../../JSTQL/src/language/generated/ast";
 
 export async function parseJSTQL(jstql: string): Promise<LocalProp[]> {
     let model: Model = await parseDSLtoAST(jstql);
-    let proposals: LocalProp[] = [];
+    let localProposals: LocalProp[] = [];
     for (let proposal of model.proposals) {
-        let pairs: TransformRecipe[] = [];
+        let cases: TransformRecipe[] = [];
-        for (let pair of proposal.pair) {
+
-            pairs.push({
+        for (let singleCase of proposal.case) {
-                applicableTo: pair.aplTo.apl_to_code,
+            cases.push({
-                transformTo: pair.traTo.transform_to_code,
+                applicableTo: singleCase.aplTo.apl_to_code,
+                transformTo: singleCase.traTo.transform_to_code,
             });
         }
-        proposals.push({ pairs });
+        localProposals.push({ cases });
     }
-    return proposals;
+    return localProposals;
 }

src/matcher/matcher.ts

@@ -14,15 +14,16 @@ export interface MatchedTreeNode {
 
 export interface PairedNodes {
     aplToNode: t.Node;
-    codeNode: t.Node;
+    codeNode: t.Node[];
 }
 
 export interface Match {
     statements: TreeNode<PairedNodes>[];
 }
 
-enum MatchCurrentResult {
+enum MatchResult {
     MatchedWithWildcard,
+    MatchedWithStarredWildcard,
     Matched,
     NoMatch,
 }
@@ -73,14 +74,17 @@ export class Matcher {
     singleExprMatcher(
         code: TreeNode<t.Node>,
         aplTo: TreeNode<t.Node>
-    ): TreeNode<PairedNodes> | undefined {
+    ): [TreeNode<PairedNodes> | undefined, MatchResult] {
         // If we are at start of ApplicableTo, start a new search on each of the child nodes
         if (aplTo.element === this.aplToFull) {
             // Perform a new search on all child nodes before trying to verify current node
             let temp = [];
             // If any matches bubble up from child nodes, we have to store it
             for (let code_child of code.children) {
-                let maybeChildMatch = this.singleExprMatcher(code_child, aplTo);
+                let [maybeChildMatch, matchResult] = this.singleExprMatcher(
+                    code_child,
+                    aplTo
+                );
                 if (maybeChildMatch) {
                     temp.push(maybeChildMatch);
                 }
@@ -98,59 +102,102 @@ export class Matcher {
 
         let curMatches = this.checkCodeNode(code.element, aplTo.element);
         let pairedCurrent: TreeNode<PairedNodes> = new TreeNode(null, {
-            codeNode: code.element,
+            codeNode: [code.element],
             aplToNode: aplTo.element,
         });
-        if (curMatches === MatchCurrentResult.NoMatch) {
+        if (curMatches === MatchResult.NoMatch) {
-            return;
+            return [undefined, MatchResult.NoMatch];
-        } else if (curMatches === MatchCurrentResult.MatchedWithWildcard) {
+        } else if (
-            return pairedCurrent;
+            curMatches === MatchResult.MatchedWithWildcard ||
-        } else if (code.children.length !== aplTo.children.length) {
+            curMatches === MatchResult.MatchedWithStarredWildcard
-            return;
+        ) {
+            return [pairedCurrent, curMatches];
         }
         // At this point current does match
         // Perform a search on each of the children of both AplTo and Code.
 
-        for (let i = 0; i < aplTo.children.length; i++) {
+        let i = 0;
-            let childSearch = this.singleExprMatcher(
+        let aplToi = 0;
+        while (aplToi < aplTo.children.length) {
+            if (i >= code.children.length) {
+                return [undefined, MatchResult.NoMatch];
+            }
+            let [pairedChild, childResult] = this.singleExprMatcher(
                 code.children[i],
-                aplTo.children[i]
+                aplTo.children[aplToi]
             );
 
-            if (childSearch === undefined) {
+            if (pairedChild === undefined) {
                 // Failed to get a full match, so early return here
-                return;
+                return [undefined, MatchResult.NoMatch];
-            }
-            childSearch.parent = pairedCurrent;
-            pairedCurrent.children.push(childSearch);
             }
 
+            pairedChild.parent = pairedCurrent;
+            pairedCurrent.children.push(pairedChild);
+            if (childResult === MatchResult.MatchedWithStarredWildcard) {
+                i += 1;
+                while (i < code.children.length) {
+                    let [maybeChild, starChildResult] = this.singleExprMatcher(
+                        code.children[i],
+                        aplTo.children[aplToi]
+                    );
+                    if (
+                        starChildResult !=
+                            MatchResult.MatchedWithStarredWildcard ||
+                        maybeChild === undefined
+                    ) {
+                        i -= 1;
+                        break;
+                    }
+                    pairedChild.element.codeNode.push(
+                        ...maybeChild.element.codeNode
+                    );
+                    i += 1;
+                }
+            }
+
+            i += 1;
+            aplToi += 1;
+        }
+        if (i < code.children.length) {
+            return [undefined, MatchResult.NoMatch];
+        }
         // If we are here, a full match has been found
-        return pairedCurrent;
+        return [pairedCurrent, curMatches];
     }
 
-    private checkCodeNode(
+    private checkCodeNode(codeNode: t.Node, aplToNode: t.Node): MatchResult {
-        codeNode: t.Node,
-        aplToNode: t.Node
-    ): MatchCurrentResult {
         // First verify the internal DSL variables
-
+        if (
+            aplToNode.type === "ExpressionStatement" &&
+            aplToNode.expression.type === "Identifier"
+        ) {
+            aplToNode = aplToNode.expression;
+        }
         if (aplToNode.type === "Identifier") {
             for (let wildcard of this.internals) {
-                if (WildcardEvalVisitor.visit(wildcard.expr, codeNode)) {
+                if (aplToNode.name === wildcard.identifier.name) {
-                    return MatchCurrentResult.MatchedWithWildcard;
+                    let visitorResult = WildcardEvalVisitor.visit(
+                        wildcard.expr,
+                        codeNode
+                    );
+                    if (visitorResult && wildcard.star) {
+                        return MatchResult.MatchedWithStarredWildcard;
+                    } else if (visitorResult) {
+                        return MatchResult.MatchedWithWildcard;
+                    }
                 }
             }
         }
 
         if (codeNode.type != aplToNode.type) {
-            return MatchCurrentResult.NoMatch;
+            return MatchResult.NoMatch;
         }
 
         //If not an internal DSL variable, gotta verify that the identifier is the same
         if (codeNode.type === "Identifier" && aplToNode.type === "Identifier") {
             if (codeNode.name != aplToNode.name) {
-                return MatchCurrentResult.NoMatch;
+                return MatchResult.NoMatch;
             }
         }
         for (let key of Object.keys(aplToNode)) {
@@ -159,11 +206,11 @@ export class Matcher {
             }
 
             if (!Object.keys(codeNode).includes(key)) {
-                return MatchCurrentResult.NoMatch;
+                return MatchResult.NoMatch;
             }
         }
 
-        return MatchCurrentResult.Matched;
+        return MatchResult.Matched;
     }
 
     multiStatementMatcher(code: TreeNode<t.Node>, aplTo: TreeNode<t.Node>) {
@@ -184,16 +231,43 @@ export class Matcher {
         for (let y = 0; y <= code.length - aplTo.length; y++) {
             let fullMatch = true;
             let statements: TreeNode<PairedNodes>[] = [];
-            for (let i = 0; i < aplTo.length; i++) {
+            let aplToi = 0;
-                let res = this.exactExprMatcher(code[i + y], aplTo[i]);
+            let codei = 0;
-                if (!res) {
+            while (aplToi < aplTo.length) {
+                let [paired, matchResult] = this.exactExprMatcher(
+                    code[codei + y],
+                    aplTo[aplToi]
+                );
+                if (!paired) {
                     fullMatch = false;
                     break;
                 }
-                statements.push(res);
+
+                if (matchResult === MatchResult.MatchedWithStarredWildcard) {
+                    codei += 1;
+                    while (codei + y < code.length) {
+                        let [next, nextMatchRes] = this.exactExprMatcher(
+                            code[codei + y],
+                            aplTo[aplToi]
+                        );
+                        if (
+                            !next ||
+                            nextMatchRes !==
+                                MatchResult.MatchedWithStarredWildcard
+                        ) {
+                            codei -= 1;
+                            break;
+                        }
+                        paired.element.codeNode.push(...next.element.codeNode);
+                        codei += 1;
+                    }
+                }
+
+                statements.push(paired);
+                aplToi += 1;
+                codei += 1;
             }
             if (fullMatch) {
-                console.log(statements.length);
                 this.matches.push({ statements });
             }
         }
@@ -201,34 +275,68 @@ export class Matcher {
     exactExprMatcher(
         code: TreeNode<t.Node>,
         aplTo: TreeNode<t.Node>
-    ): TreeNode<PairedNodes> | undefined {
+    ): [TreeNode<PairedNodes> | undefined, MatchResult] {
-        let curMatches =
+        let curMatches = this.checkCodeNode(code.element, aplTo.element);
-            this.checkCodeNode(code.element, aplTo.element) &&
-            code.children.length >= aplTo.children.length;
 
-        if (!curMatches) {
+        if (curMatches === MatchResult.NoMatch) {
-            return undefined;
+            return [undefined, MatchResult.NoMatch];
         }
 
         let paired: TreeNode<PairedNodes> = new TreeNode(null, {
             aplToNode: aplTo.element,
-            codeNode: code.element,
+            codeNode: [code.element],
         });
+        if (
+            curMatches === MatchResult.MatchedWithStarredWildcard ||
+            curMatches === MatchResult.MatchedWithWildcard
+        ) {
+            return [paired, curMatches];
+        }
 
-        for (let i = 0; i < aplTo.children.length; i++) {
+        let i = 0;
-            let childRes = this.exactExprMatcher(
+        let aplToi = 0;
+        while (i < code.children.length && aplToi < aplTo.children.length) {
+            let [pairedChild, childResult] = this.exactExprMatcher(
                 code.children[i],
-                aplTo.children[i]
+                aplTo.children[aplToi]
             );
-            if (!childRes) {
+            if (!pairedChild) {
                 // If child is not match the entire thing is not a match;
-                return undefined;
+                return [undefined, MatchResult.NoMatch];
-            }
-            // This is a match, so we store it
-            childRes.parent = paired;
-            paired.children.push(childRes);
             }
 
-        return paired;
+            // This is a match, so we store it
+            pairedChild.parent = paired;
+            paired.children.push(pairedChild);
+
+            if (childResult === MatchResult.MatchedWithStarredWildcard) {
+                i += 1;
+                while (i < code.children.length) {
+                    let [maybeChild, starChildResult] = this.singleExprMatcher(
+                        code.children[i],
+                        aplTo.children[aplToi]
+                    );
+                    if (
+                        starChildResult !=
+                            MatchResult.MatchedWithStarredWildcard ||
+                        maybeChild === undefined
+                    ) {
+                        i -= 1;
+                        break;
+                    }
+                    pairedChild.element.codeNode.push(
+                        ...maybeChild.element.codeNode
+                    );
+                    i += 1;
+                }
+            }
+
+            i += 1;
+            aplToi += 1;
+        }
+        if (i < code.children.length) {
+            return [undefined, MatchResult.NoMatch];
+        }
+        return [paired, curMatches];
     }
 }

src/matcher/wildcardEvaluator.ts

@@ -7,7 +7,9 @@ import {
     UnaryExpr,
     Wildcard,
     WildcardNode,
+    WildcardParser,
 } from "../parser/parse";
+import { WildcardTokenizer } from "../parser/wildcardTokenizer";
 
 export class WildcardEvalVisitor {
     static visit(node: WildcardNode, toComp: t.Node): boolean {
@@ -34,9 +36,26 @@ export class WildcardEvalVisitor {
                 let cur = node as Identifier;
                 if (cur.name === "Expression") {
                     return t.isExpression(toComp);
+                } else if (cur.name === "Statement") {
+                    return t.isStatement(toComp);
                 }
                 return cur.name === toComp.type;
             }
         }
     }
 }
+
+function testWildcardEval() {
+    console.log(
+        WildcardEvalVisitor.visit(
+            new WildcardParser(
+                new WildcardTokenizer(
+                    "statements:(Statement && !ReturnStatement)*"
+                ).tokenize()
+            ).parse().expr,
+            t.variableDeclaration("let", [
+                t.variableDeclarator(t.identifier("Id"), null),
+            ])
+        )
+    );
+}

src/parser/parse.ts

@@ -275,7 +275,15 @@ export function parse_with_plugins(
     code: string
 ): babelparser.ParseResult<t.File> {
     return babelparser.parse(code, {
-        plugins: [["pipelineOperator", { proposal: "hack", topicToken: "%" }]],
+        plugins: [
+            ["pipelineOperator", { proposal: "hack", topicToken: "%" }],
+            "doExpressions",
+            "topLevelAwait",
+        ],
+        allowAwaitOutsideFunction: true,
+        allowReturnOutsideFunction: true,
+        allowUndeclaredExports: true,
+        sourceType: "unambiguous",
     });
 }
 

src/test/test_outputs/awaitToPromise_output.js

@@ -0,0 +1,9 @@
+async function something() {
+  let a = 100;
+  a *= 100000;
+  return fetch("https://uib.no").then(uib => {
+    a += 100000;
+    a -= 1000;
+    [a, uib];
+  });
+}

src/test/test_outputs/do_output.js

@@ -0,0 +1,12 @@
+let aaaa = do {
+  let g = 100;
+  let ff = 10;
+  let ggg = a(b);
+  100;
+};
+let bbaaa = do {
+  let lllll = 1 + 1;
+  100 + 100;
+  const aaaaa = aaaa(bb);
+  lllll;
+};

src/test/test_outputs/pipeline_output.js

@@ -0,0 +1,4 @@
+a |> w(%) |> w(%) |> w(%) |> w(%) |> w(%) |> w(%) |> w(%) |> w(%) |> w(%) |> w(%);
+a |> b(%, a |> b(%, a |> b(%, a |> b(%, a |> b(%, a |> b(%, a |> b(%, b)))))));
+a |> b(%, a) |> b(%, a) |> b(%, a) |> b(%, a) |> b(%, a) |> b(%, a) |> b(%, a) |> b(%, a) |> b(%, a) |> b(%, a) |> b(%, a) |> b(%, a);
+b(b(b(b(a, a, a), a, a), a, a), a, a);

src/test/test_transform.test.ts

@@ -1,39 +1,44 @@
 import { expect, test } from "bun:test";
 import { TransformRecipe, transform } from "../transform/transform";
+import { parseJSTQL } from "../langium/langiumRunner";
 
-const transformExample: TransformRecipe = {
+async function runTest(inputJS: string, inputJSTQL: string): Promise<string> {
-    applicableTo: `<<a>>(<<b:Identifier|Expression>>);`,
+    //transform(selfHostedTransformExampleMultiStmt, codeFromFile);
-    transformTo: "<<b>> |> <<a>>(%)",
+    const file = Bun.file(inputJS);
-};
+    const codeFromFile = await file.text();
-const code =
-    "a(something);a(1+1);something(some_other_thing + 1 + 10 + 100); console.log(a)";
 
-test("Test code: " + code + " on " + transformExample.applicableTo, () => {
+    const test_file = Bun.file(inputJSTQL);
-    expect(transform(transformExample, code).length).toBe(
+    const test_JSTQL = await test_file.text();
-        "something |> a(%);1 + 1 |> a(%);some_other_thing + 1 + 10 + 100 |> something(%);console.log(a);"
+    let proposals = await parseJSTQL(test_JSTQL);
+
+    let code = transform(proposals[0].cases, codeFromFile);
+    return code;
+}
+let pipelineRes = await runTest(
+    "test_files/pipeline_test.js",
+    "dsl_files/pipeline.jstql"
 );
+let pipelineResFile = await Bun.file(
+    "src/test/test_outputs/pipeline_output.js"
+).text();
+test("Test code: pipeline", () => {
+    expect(pipelineRes).toBe(pipelineResFile);
 });
-// Expected outcome: 3 correct matches
+let doRes = await runTest("test_files/do_test.js", "dsl_files/do.jstql");
-const secondTransformExample: TransformRecipe = {
+
-    applicableTo: `<<a>>.<<b>>(<<c:Expression|Identifier>>);`,
+let doResFile = await Bun.file("src/test/test_outputs/do_output.js").text();
-    transformTo: "c |> a.b(%);",
+test("Test code: do", () => {
-};
+    expect(doRes).toBe(doResFile);
-const code2 = `console.log(a);something.sometingOther(b(c));some.thing(1+1); a(b)`;
+});
-test(
+
-    "Test code: " + code2 + " on " + secondTransformExample.applicableTo,
+let awaitToPromise = await runTest(
-    () => {
+    "test_files/do_test.js",
-        expect(transform(secondTransformExample, code2).length).toBe(3);
+    "dsl_files/do.jstql"
-    }
-);
-// Expected outcome: 1 correct match
-const thirdTransformExample: TransformRecipe = {
-    applicableTo: `myFunction(<<a:Expression|Identifier>>)`,
-    transformTo: `a |> myFunction(%)`,
-};
-const code3 = `myFunction(a);otherFunction(a); myFunction.otherfunction(a)`;
-test(
-    "Test code: " + code3 + " on " + thirdTransformExample.applicableTo,
-    () => {
-        expect(transform(thirdTransformExample, code3)).toBe(``);
-    }
 );
+
+let awaitToPromiseOutput = await Bun.file(
+    "src/test/test_outputs/do_output.js"
+).text();
+test("Test code: do", () => {
+    expect(awaitToPromise).toBe(awaitToPromiseOutput);
+});

src/transform/transform.ts

@@ -2,8 +2,7 @@ import traverse from "@babel/traverse";
 import * as t from "@babel/types";
 import generate from "@babel/generator";
 import {
-    InternalDSLVariable,
+    Wildcard,
-    parseInternal,
     parseInternalAplTo,
     parseInternalTraTo,
     parse_with_plugins,
@@ -19,7 +18,7 @@ import { transformMatch, transformer } from "./transformMatch";
 import { preludeBuilder } from "../parser/preludeBuilder";
 import * as babelparser from "@babel/parser";
 export interface Proposal {
-    pairs: TransformRecipe[];
+    cases: TransformRecipe[];
 }
 
 export interface TransformRecipe {
@@ -29,16 +28,19 @@ export interface TransformRecipe {
 export interface SelfHostedRecipe extends TransformRecipe {
     prelude: string;
 }
-export function transform(recipe: TransformRecipe, code: string): string {
+export function transform(recipes: TransformRecipe[], code: string): string {
+    let codeAST: t.Node = parse_with_plugins(code);
+
+    for (let recipe of recipes) {
         if ((<SelfHostedRecipe>recipe).prelude !== undefined) {
             // We are using the self hosted version
-        return transformSelfHosted(
+            codeAST = transformSelfHosted(
                 {
                     applicableTo: recipe.applicableTo,
                     transformTo: recipe.transformTo,
                 },
                 preludeBuilder((recipe as SelfHostedRecipe).prelude),
-            code
+                codeAST
             );
         } else {
             // We are using JSTQL
@@ -49,22 +51,25 @@ export function transform(recipe: TransformRecipe, code: string): string {
             );
             let transformTo = parseInternalTraTo(recipe.transformTo);
 
-        return transformSelfHosted(
+            codeAST = transformSelfHosted(
                 { applicableTo, transformTo },
                 prelude,
-            code
+                codeAST
             );
         }
     }
 
+    let output = generate(codeAST, { topicToken: "%" }).code;
+    //showTree(transformToTree);
+    return output;
+}
+
 function transformSelfHosted(
     recipe: TransformRecipe,
-    internals: InternalDSLVariable,
+    internals: Wildcard[],
-    code: string
+    codeAST: t.Node
-): string {
+): t.Node {
-    console.log(recipe);
+    let codeTree = makeTree(codeAST as babelparser.ParseResult<t.File>);
-    let codeAST = parse_with_plugins(code);
-    let codeTree = makeTree(codeAST);
     let applicabelToAST = parse_with_plugins(recipe.applicableTo);
 
     let applicableToTree = makeTree(applicabelToAST);
@@ -78,15 +83,9 @@ function transformSelfHosted(
     ) {
         throw new Error("This no worky LOL");
     }
-    showTree(applicableToTree);
-    console.log(generate(codeAST));
 
     let matches = runMatch(codeTree, applicableToTree, internals);
-    console.log(matches.length);
+    console.log("We found", matches.length, "matches");
-
     let outputAST = transformer(matches, transformToTree, codeAST, transformTo);
-
+    return outputAST;
-    let output = generate(outputAST, { topicToken: "%" }).code;
-    //showTree(transformToTree);
-    return output;
 }

src/transform/transformMatch.ts

@@ -34,13 +34,15 @@ export function transformer(
                         )
                     ) {
                         if (
-                            path.node === match.statements[0].element.codeNode
+                            path.node ===
+                            match.statements[0].element.codeNode[0]
                         ) {
                             path.replaceWithMultiple(
                                 traToWithWildcards.program.body
                             );
                             let siblings = path.getAllNextSiblings();
 
+                            // For multi line applicable to
                             for (
                                 let i = 0;
                                 i < match.statements.length - 1;
@@ -48,6 +50,30 @@ export function transformer(
                             ) {
                                 siblings[i].remove();
                             }
+
+                            // For when we have matched with *
+
+                            for (let matchStmt of match.statements) {
+                                for (let [
+                                    i,
+                                    stmtMatchedWithStar,
+                                ] of matchStmt.element.codeNode.entries()) {
+                                    let siblingnodes = siblings.map(
+                                        (a) => a.node
+                                    );
+                                    if (
+                                        siblingnodes.includes(
+                                            stmtMatchedWithStar
+                                        )
+                                    ) {
+                                        let index =
+                                            siblingnodes.indexOf(
+                                                stmtMatchedWithStar
+                                            );
+                                        siblings[index].remove();
+                                    }
+                                }
+                            }
                         }
                     }
                 },
@@ -76,6 +102,19 @@ export function transformMatch(
                     path.replaceWithMultiple(match.element.codeNode);
                 }
             },
+            ExpressionStatement: (path) => {
+                if (trnTo.element.type === "ExpressionStatement") {
+                    if (
+                        path.node.expression.type === "Identifier" &&
+                        trnTo.element.expression.type === "Identifier"
+                    ) {
+                        let ident = path.node.expression;
+                        if (ident.name === trnTo.element.expression.name) {
+                            path.replaceWithMultiple(match.element.codeNode);
+                        }
+                    }
+                }
+            },
         });
     } else {
         for (let match_child of match.children) {
@@ -90,6 +129,18 @@ export function transformMatch(
 function matchNode(aplTo: t.Node, trnTo: t.Node): boolean {
     //console.log(trnTo);
 
+    if (
+        trnTo.type === "ExpressionStatement" &&
+        aplTo.type == "ExpressionStatement"
+    ) {
+        if (
+            trnTo.expression.type === "Identifier" &&
+            aplTo.expression.type === "Identifier"
+        ) {
+            return aplTo.expression.name === trnTo.expression.name;
+        }
+    }
+
     if (trnTo.type == "Identifier" && aplTo.type == "Identifier") {
         return aplTo.name === trnTo.name;
     }

test_files/awaitToPromise.js

@@ -0,0 +1,8 @@
+async function something() {
+    let a = 100;
+    a *= 100000;
+    let uib = await fetch("https://uib.no");
+    a += 100000;
+    a -= 1000;
+    return [a, uib];
+}

test_files/do_test.js

@@ -0,0 +1,13 @@
+let aaaa = () => {
+    let g = 100;
+    let ff = 10;
+    let ggg = a(b);
+    return 100;
+};
+
+var bbaaa = (function () {
+    let lllll = 1 + 1;
+    100 + 100;
+    const aaaaa = aaaa(bb);
+    return lllll;
+})();

test_files/pipeline_test.js

@@ -0,0 +1,7 @@
+w(w(w(w(w(w(w(w(w(w(a))))))))));
+
+b(a, b(a, b(a, b(a, b(a, b(a, b(a, b)))))));
+
+b(b(b(b(b(b(b(b(b(b(b(b(a, a), a), a), a), a), a), a), a), a), a), a), a);
+
+b(b(b(b(a, a, a), a, a), a, a), a, a);

test_files/star_test.js

@@ -0,0 +1,4 @@
+let x = () => {
+    let b = 0;
+    return b;
+};