polsevev/JSTQL-JS-Transform
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

optimized transformer and created evaluation

Browse source
This commit is contained in:
Rolf Martin Glomsrud 2024-05-23 13:46:40 +02:00
parent 7a1f099b33
commit f7d7b1c6d3
6 changed files with 195 additions and 68 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
output_files/pipeline_out.js Normal file
View file

@ -0,0 +1,4 @@
a |> w(%) |> w(%) |> w(%) |> w(%) |> w(%) |> w(%) |> w(%) |> w(%) |> w(%) |> w(%);
a |> b(%, a |> b(%, a |> b(%, a |> b(%, a |> b(%, a |> b(%, a |> b(%, b)))))));
a |> b(%, a) |> b(%, a) |> b(%, a) |> b(%, a) |> b(%, a) |> b(%, a) |> b(%, a) |> b(%, a) |> b(%, a) |> b(%, a) |> b(%, a) |> b(%, a);
b(b(b(b(a, a, a), a, a), a, a), a, a);

50
src/index.ts
View file

@ -7,25 +7,65 @@ import {
TransformRecipe,
transform,
} from "./transform/transform";
import { readdir } from "node:fs/promises";
import { parseJSTQL } from "./langium/langiumRunner";
const dir = "../prettier/src";
const path = "test_files/test2.js";
const path = "test_files/pipeline_test.js";
const file = Bun.file(path);
const codeFromFile = await file.text();
const main = async () => {
//transform(selfHostedTransformExampleMultiStmt, codeFromFile);
/*
console.log(codeFromFile);
const jstql_file =
"/home/rolfmg/Coding/Master/didactic-chainsaw/dsl_files/awaitToPromise.jstql";
"/home/rolfmg/Coding/Master/didactic-chainsaw/dsl_files/pipeline.jstql";
const test_file = Bun.file(jstql_file);
const test_JSTQL = await test_file.text();
let proposals = await parseJSTQL(test_JSTQL);
let code = transform(proposals[0].cases, codeFromFile);
await Bun.write("output_files/test2.js", code);
let [code, count] = transform(proposals[0].cases, codeFromFile);
await Bun.write("output_files/pipeline_out.js", code);
return;
*/
const jstql_file =
"/home/rolfmg/Coding/Master/didactic-chainsaw/dsl_files/pipeline.jstql";
const test_file = Bun.file(jstql_file);
const test_JSTQL = await test_file.text();
let proposals = await parseJSTQL(test_JSTQL);
let basepathExamplesJSFiles = "../next.js";
let examples = (await readdir(basepathExamplesJSFiles, { recursive: true }))
.filter((x) => x.endsWith(".js"))
.map((x) => basepathExamplesJSFiles + "/" + x);
console.log(examples);
let sum = 0;
console.log("Scripts found ", sum, "matches!");
let count = 0;
for (let examplesFile of examples) {
try {
if (examplesFile.split("/").includes("compiled")) {
continue;
}
console.log(examplesFile);
let script = await Bun.file(examplesFile).text();
let [resultString, matches] = transform(proposals[0].cases, script);
sum += matches;
if (matches > 0) {
await Bun.write(
"output_testing/" + count + examplesFile.split("/").at(-1),
resultString
);
count += 1;
}
} catch (e) {
console.log("Failed");
}
console.log("current sum", sum);
}
console.log(sum);
};
main();

1
src/parser/parse.ts
View file

@ -74,7 +74,6 @@ export function parseInternalAplTo(code: string): InternalParseResult {
cleanedJS += code[i];
}
}
console.log(prelude, cleanedJS);
return { prelude, cleanedJS };
}
export interface Identifier extends WildcardNode {

6
src/test/test_transform.test.ts
View file

@ -22,13 +22,13 @@ let pipelineResFile = await Bun.file(
"src/test/test_outputs/pipeline_output.js"
).text();
test("Test code: pipeline", () => {
expect(pipelineRes).toBe(pipelineResFile);
expect(pipelineRes).toEqual([pipelineResFile, 29]);
});
let doRes = await runTest("test_files/do_test.js", "dsl_files/do.jstql");
let doResFile = await Bun.file("src/test/test_outputs/do_output.js").text();
test("Test code: do", () => {
expect(doRes).toBe(doResFile);
expect(doRes).toEqual([doResFile, 2]);
});
let awaitToPromise = await runTest(
@ -40,5 +40,5 @@ let awaitToPromiseOutput = await Bun.file(
"src/test/test_outputs/awaitToPromise_output.js"
).text();
test("Test code: await to promise", () => {
expect(awaitToPromise).toBe(awaitToPromiseOutput);
expect(awaitToPromise).toEqual([awaitToPromiseOutput, 1]);
});

34
src/transform/transform.ts
View file

@ -28,13 +28,16 @@ export interface TransformRecipe {
export interface SelfHostedRecipe extends TransformRecipe {
prelude: string;
}
export function transform(recipes: TransformRecipe[], code: string): string {
export function transform(
recipes: TransformRecipe[],
code: string
): [string, number] {
let codeAST: t.Node = parse_with_plugins(code);
let amount = 0;
for (let recipe of recipes) {
if ((<SelfHostedRecipe>recipe).prelude !== undefined) {
// We are using the self hosted version
codeAST = transformSelfHosted(
let temp = transformSelfHosted(
{
applicableTo: recipe.applicableTo,
transformTo: recipe.transformTo,
@ -42,6 +45,8 @@ export function transform(recipes: TransformRecipe[], code: string): string {
preludeBuilder((recipe as SelfHostedRecipe).prelude),
codeAST
);
codeAST = temp[0];
amount += temp[1];
} else {
// We are using JSTQL
// We have to parse JSTQL to the self hosted version
@ -51,24 +56,26 @@ export function transform(recipes: TransformRecipe[], code: string): string {
);
let transformTo = parseInternalTraTo(recipe.transformTo);
codeAST = transformSelfHosted(
let temp = transformSelfHosted(
{ applicableTo, transformTo },
prelude,
codeAST
);
codeAST = temp[0];
amount += temp[1];
}
}
let output = generate(codeAST, { topicToken: "%" }).code;
//showTree(transformToTree);
return output;
return [output, amount];
}
function transformSelfHosted(
recipe: TransformRecipe,
internals: Wildcard[],
codeAST: t.Node
): t.Node {
): [t.Node, number] {
let codeTree = makeTree(codeAST as babelparser.ParseResult<t.File>);
let applicabelToAST = parse_with_plugins(recipe.applicableTo);
@ -83,9 +90,18 @@ function transformSelfHosted(
) {
throw new Error("This no worky LOL");
}
let matches = runMatch(codeTree, applicableToTree, internals);
console.log("We found", matches.length, "matches");
let outputAST = transformer(matches, transformToTree, codeAST, transformTo);
return outputAST;
let outputAST = transformer(
matches,
transformToTree,
codeAST,
transformTo,
internals.map((x) => x.identifier.name)
);
console.log("Finished transforming");
return [outputAST, matches.length];
}

168
src/transform/transformMatch.ts
View file

@ -12,78 +12,146 @@ import { Match, MatchedTreeNode, PairedNodes } from "../matcher/matcher";
import traverse from "@babel/traverse";
import generate from "@babel/generator";
import { TransformRecipe } from "./transform";
import { Wildcard } from "../parser/parse";
export function transformer(
matches: Match[],
transformTo: TreeNode<t.Node>,
codeAST: t.Node,
traToAST: t.File
traToAST: t.File,
wildcardIdentifiers: string[]
): t.Node {
let transformedTransformTo: [Match, t.File][] = [];
for (let match of matches.reverse()) {
try {
let traToWithWildcards = structuredClone(traToAST);
for (let match_stmt of match.statements) {
let aplToWildcards: Map<string, t.Node[]> = new Map();
FindWildcardsAplTo(
match_stmt,
wildcardIdentifiers,
aplToWildcards
);
transformMatch(match_stmt, transformTo, traToWithWildcards);
}
traverse(codeAST, {
enter(path) {
if (
!(
path.node.type === "Program" ||
path.node.type === "File"
)
) {
if (
path.node ===
match.statements[0].element.codeNode[0]
) {
path.replaceWithMultiple(
traToWithWildcards.program.body
);
let siblings = path.getAllNextSiblings();
// For multi line applicable to
for (
let i = 0;
i < match.statements.length - 1;
i++
) {
siblings[i].remove();
}
transformedTransformTo.push([match, traToWithWildcards]);
} catch (e) {
console.log(e);
}
//console.log("Finished replacement");
}
for (let [match, traToWithWildcards] of transformedTransformTo) {
traverse(codeAST, {
enter(path) {
if (
!(path.node.type === "Program" || path.node.type === "File")
) {
if (path.node === match.statements[0].element.codeNode[0]) {
path.replaceWithMultiple(
traToWithWildcards.program.body
);
let siblings = path.getAllNextSiblings();
// For when we have matched with *
// For multi line applicable to
for (let i = 0; i < match.statements.length - 1; i++) {
siblings[i].remove();
}
for (let matchStmt of match.statements) {
for (let [
i,
stmtMatchedWithStar,
] of matchStmt.element.codeNode.entries()) {
let siblingnodes = siblings.map(
(a) => a.node
);
if (
siblingnodes.includes(
// For when we have matched with +
for (let matchStmt of match.statements) {
for (let [
i,
stmtMatchedWithStar,
] of matchStmt.element.codeNode.entries()) {
let siblingnodes = siblings.map((a) => a.node);
if (
siblingnodes.includes(stmtMatchedWithStar)
) {
let index =
siblingnodes.indexOf(
stmtMatchedWithStar
)
) {
let index =
siblingnodes.indexOf(
stmtMatchedWithStar
);
siblings[index].remove();
}
);
siblings[index].remove();
}
}
}
}
},
});
} catch (e) {
console.log(e);
}
}
},
});
}
return codeAST;
}
function FindWildcardsAplTo(
match: TreeNode<PairedNodes>,
wildcardIdentifiers: string[],
out: Map<string, t.Node[]>
) {
let ident = isWildcardIdent(wildcardIdentifiers, match.element.aplToNode);
if (!ident) {
return;
}
return codeAST;
out.set(ident, match.element.codeNode);
for (let child of match.children) {
FindWildcardsAplTo(child, wildcardIdentifiers, out);
}
}
function isWildcardIdent(
wildcards: string[],
aplToNode: t.Node
): string | undefined {
if (aplToNode.type === "Identifier") {
for (let ident of wildcards) {
if (aplToNode.name === ident) {
return ident;
}
}
} else if (
aplToNode.type === "ExpressionStatement" &&
aplToNode.expression.type === "Identifier"
) {
for (let ident of wildcards) {
if (aplToNode.expression.name === ident) {
return ident;
}
}
}
return undefined;
}
export function transformTransformTo(
wildcardsMatchedWith: Map<string, t.Node>,
output: t.Node
) {
traverse(output, {
Identifier: (path) => {
if (wildcardsMatchedWith.has(path.node.name)) {
let newNode = wildcardsMatchedWith.get(path.node.name);
if (newNode) {
path.replaceWithMultiple(newNode);
}
}
},
ExpressionStatement: (path) => {
if (
path.node.expression.type === "Identifier" &&
wildcardsMatchedWith.has(path.node.expression.name)
) {
let newNode = wildcardsMatchedWith.get(
path.node.expression.name
);
if (newNode) {
path.replaceWithMultiple(newNode);
}
}
},
});
}
export function transformMatch(