Initialized Langium, i am however having some issues with it, perhaps writing my own parser is better?

This commit is contained in:
Rolf Martin Glomsrud 2023-12-06 13:44:49 +01:00
parent 22f4b9e8f1
commit 57440fb01f
32 changed files with 3219 additions and 60 deletions

3
.gitmodules vendored
View file

@ -0,0 +1,3 @@
[submodule "babel"]
path = babel
url = git@github.com:polsevev/babel.git

1
babel Submodule

@ -0,0 +1 @@
Subproject commit 2f9c48d4eda2d69908fc53ea285f47ed2c540f7e

View file

@ -0,0 +1,13 @@
{
"root": true,
"parser": "@typescript-eslint/parser",
"parserOptions": {
"ecmaVersion": 6,
"sourceType": "module"
},
"plugins": [
"@typescript-eslint"
],
"rules": {
}
}

11
didactic-chainsaw-dsl/.gitignore vendored Normal file
View file

@ -0,0 +1,11 @@
.vscode/*
!.vscode/extensions.json
!.vscode/launch.json
!.vscode/tasks.json
node_modules/
out/
src/language/generated/
static/bundle/
static/monaco-editor-workers/
static/worker/
syntaxes/

View file

@ -0,0 +1,9 @@
{
// See https://go.microsoft.com/fwlink/?LinkId=827846 to learn about workspace recommendations.
// Extension identifier format: ${publisher}.${name}. Example: vscode.csharp
// List of extensions which should be recommended for users of this workspace.
"recommendations": [
"langium.langium-vscode"
]
}

View file

@ -0,0 +1,35 @@
// A launch configuration that launches the extension inside a new window
// Use IntelliSense to learn about possible attributes.
// Hover to view descriptions of existing attributes.
// For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
{
"version": "0.2.0",
"configurations": [
{
"name": "Run Extension",
"type": "extensionHost",
"request": "launch",
"args": [
"--extensionDevelopmentPath=${workspaceFolder}"
],
"sourceMaps": true,
"outFiles": [
"${workspaceFolder}/out/**/*.js"
]
},
{
"name": "Attach to Language Server",
"type": "node",
"port": 6009,
"request": "attach",
"skipFiles": [
"<node_internals>/**"
],
"sourceMaps": true,
"outFiles": [
"${workspaceFolder}/out/**/*.js",
"${workspaceFolder}/node_modules/langium"
]
}
]
}

View file

@ -0,0 +1,21 @@
{
// See https://go.microsoft.com/fwlink/?LinkId=733558
// for the documentation about the tasks.json format
"version": "2.0.0",
"tasks": [
{
"label": "Build js-transform-lang",
"command": "npm run langium:generate && npm run build",
"type": "shell",
"group": {
"kind": "build",
"isDefault": true
},
"detail": "Langium: Generate grammar and build the js-transform-lang language",
"icon": {
"color": "terminal.ansiGreen",
"id": "server-process"
}
}
]
}

View file

@ -0,0 +1,4 @@
.vscode/**
.vscode-test/**
.gitignore
langium-quickstart.md

View file

@ -0,0 +1,4 @@
#!/usr/bin/env node
import main from '../out/cli/main.js';
main();

View file

@ -0,0 +1,54 @@
//@ts-check
import * as esbuild from 'esbuild';
const watch = process.argv.includes('--watch');
const minify = process.argv.includes('--minify');
const success = watch ? 'Watch build succeeded' : 'Build succeeded';
function getTime() {
const date = new Date();
return `[${`${padZeroes(date.getHours())}:${padZeroes(date.getMinutes())}:${padZeroes(date.getSeconds())}`}] `;
}
function padZeroes(i) {
return i.toString().padStart(2, '0');
}
const plugins = [{
name: 'watch-plugin',
setup(build) {
build.onEnd(result => {
if (result.errors.length === 0) {
console.log(getTime() + success);
}
});
},
}];
const ctx = await esbuild.context({
// Entry points for the vscode extension and the language server
entryPoints: ['src/extension/main.ts', 'src/language/main.ts'],
outdir: 'out',
bundle: true,
target: "ES2017",
// VSCode's extension host is still using cjs, so we need to transform the code
format: 'cjs',
// To prevent confusing node, we explicitly use the `.cjs` extension
outExtension: {
'.js': '.cjs'
},
loader: { '.ts': 'ts' },
external: ['vscode'],
platform: 'node',
sourcemap: !minify,
minify,
plugins
});
if (watch) {
await ctx.watch();
} else {
await ctx.rebuild();
ctx.dispose();
}

View file

@ -0,0 +1,14 @@
{
"projectName": "JsTransformLang",
"languages": [
{
"id": "js-transform-lang",
"grammar": "src/language/js-transform-lang.langium",
"fileExtensions": [".jstl"],
"textMate": {
"out": "syntaxes/js-transform-lang.tmLanguage.json"
}
}
],
"out": "src/language/generated"
}

View file

@ -0,0 +1,40 @@
# Welcome to your Langium VS Code Extension
## What's in the folder
This folder contains all necessary files for your language extension.
* `package.json` - the manifest file in which you declare your language support.
* `language-configuration.json` - the language configuration used in the VS Code editor, defining the tokens that are used for comments and brackets.
* `src/extension/main.ts` - the main code of the extension, which is responsible for launching a language server and client.
* `src/language/js-transform-lang.langium` - the grammar definition of your language.
* `src/language/main.ts` - the entry point of the language server process.
* `src/language/js-transform-lang-module.ts` - the dependency injection module of your language implementation. Use this to register overridden and added services.
* `src/language/js-transform-lang-validator.ts` - an example validator. You should change it to reflect the semantics of your language.
* `src/cli/main.ts` - the entry point of the command line interface (CLI) of your language.
* `src/cli/generator.ts` - the code generator used by the CLI to write output files from DSL documents.
* `src/cli/cli-util.ts` - utility code for the CLI.
## Get up and running straight away
* Run `npm run langium:generate` to generate TypeScript code from the grammar definition.
* Run `npm run build` to compile all TypeScript code.
* Press `F5` to open a new window with your extension loaded.
* Create a new file with a file name suffix matching your language.
* Verify that syntax highlighting, validation, completion etc. are working as expected.
* Run `node ./bin/cli` to see options for the CLI; `node ./bin/cli generate <file>` generates code for a given DSL file.
## Make changes
* Run `npm run watch` to have the TypeScript compiler run automatically after every change of the source files.
* Run `npm run langium:watch` to have the Langium generator run automatically after every change of the grammar declaration.
* You can relaunch the extension from the debug toolbar after making changes to the files listed above.
* You can also reload (`Ctrl+R` or `Cmd+R` on Mac) the VS Code window with your extension to load your changes.
## Install your extension
* To start using your extension with VS Code, copy it into the `<user home>/.vscode/extensions` folder and restart Code.
* To share your extension with the world, read the [VS Code documentation](https://code.visualstudio.com/api/working-with-extensions/publishing-extension) about publishing an extension.
## To Go Further
Documentation about the Langium framework is available at https://langium.org

View file

@ -0,0 +1,30 @@
{
"comments": {
// symbol used for single line comment. Remove this entry if your language does not support line comments
"lineComment": "//",
// symbols used for start and end a block comment. Remove this entry if your language does not support block comments
"blockComment": [ "/*", "*/" ]
},
// symbols used as brackets
"brackets": [
["{", "}"],
["[", "]"],
["(", ")"]
],
// symbols that are auto closed when typing
"autoClosingPairs": [
["{", "}"],
["[", "]"],
["(", ")"],
["\"", "\""],
["'", "'"]
],
// symbols that can be used to surround a selection
"surroundingPairs": [
["{", "}"],
["[", "]"],
["(", ")"],
["\"", "\""],
["'", "'"]
]
}

2502
didactic-chainsaw-dsl/package-lock.json generated Normal file

File diff suppressed because it is too large Load diff

View file

@ -0,0 +1,70 @@
{
"name": "didactic-chainsaw-dsl",
"description": "Please enter a brief description here",
"version": "0.0.1",
"files": [
"bin"
],
"type": "module",
"scripts": {
"build": "tsc -b tsconfig.json && node esbuild.mjs",
"watch": "concurrently -n tsc,esbuild -c blue,yellow \"tsc -b tsconfig.json --watch\" \"node esbuild.mjs --watch\"",
"lint": "eslint src --ext ts",
"langium:generate": "langium generate",
"langium:watch": "langium generate --watch",
"vscode:prepublish": "npm run build && npm run lint"
},
"dependencies": {
"langium": "~2.1.0",
"vscode-languageclient": "~9.0.1",
"vscode-languageserver": "~9.0.1",
"chalk": "~5.3.0",
"commander": "~11.0.0"
},
"devDependencies": {
"@types/node": "~16.18.41",
"@typescript-eslint/parser": "~6.4.1",
"@typescript-eslint/eslint-plugin": "~6.4.1",
"eslint": "~8.47.0",
"langium-cli": "~2.1.0",
"typescript": "~5.1.6",
"@types/vscode": "~1.67.0",
"concurrently": "~8.2.1",
"esbuild": "~0.19.2"
},
"displayName": "didactic-chainsaw-dsl",
"engines": {
"vscode": "^1.67.0",
"node": ">=16.0.0"
},
"categories": [
"Programming Languages"
],
"contributes": {
"languages": [
{
"id": "js-transform-lang",
"aliases": [
"js-transform-lang",
"js-transform-lang"
],
"extensions": [".jstl"],
"configuration": "./language-configuration.json"
}
],
"grammars": [
{
"language": "js-transform-lang",
"scopeName": "source.js-transform-lang",
"path": "syntaxes/js-transform-lang.tmLanguage.json"
}
]
},
"activationEvents": [
"onLanguage:js-transform-lang"
],
"main": "./out/extension/main.cjs",
"bin": {
"js-transform-lang-cli": "./bin/cli.js"
}
}

View file

@ -0,0 +1,51 @@
import type { AstNode, LangiumDocument, LangiumServices } from 'langium';
import chalk from 'chalk';
import * as path from 'node:path';
import * as fs from 'node:fs';
import { URI } from 'langium';
export async function extractDocument(fileName: string, services: LangiumServices): Promise<LangiumDocument> {
const extensions = services.LanguageMetaData.fileExtensions;
if (!extensions.includes(path.extname(fileName))) {
console.error(chalk.yellow(`Please choose a file with one of these extensions: ${extensions}.`));
process.exit(1);
}
if (!fs.existsSync(fileName)) {
console.error(chalk.red(`File ${fileName} does not exist.`));
process.exit(1);
}
const document = services.shared.workspace.LangiumDocuments.getOrCreateDocument(URI.file(path.resolve(fileName)));
await services.shared.workspace.DocumentBuilder.build([document], { validation: true });
const validationErrors = (document.diagnostics ?? []).filter(e => e.severity === 1);
if (validationErrors.length > 0) {
console.error(chalk.red('There are validation errors:'));
for (const validationError of validationErrors) {
console.error(chalk.red(
`line ${validationError.range.start.line + 1}: ${validationError.message} [${document.textDocument.getText(validationError.range)}]`
));
}
process.exit(1);
}
return document;
}
export async function extractAstNode<T extends AstNode>(fileName: string, services: LangiumServices): Promise<T> {
return (await extractDocument(fileName, services)).parseResult?.value as T;
}
interface FilePathData {
destination: string,
name: string
}
export function extractDestinationAndName(filePath: string, destination: string | undefined): FilePathData {
filePath = path.basename(filePath, path.extname(filePath)).replace(/[.-]/g, '');
return {
destination: destination ?? path.join(path.dirname(filePath), 'generated'),
name: path.basename(filePath)
};
}

View file

@ -0,0 +1,23 @@
import type { Model } from "../language/generated/ast.js";
import * as fs from "node:fs";
import { CompositeGeneratorNode, NL, toString } from "langium";
import * as path from "node:path";
import { extractDestinationAndName } from "./cli-util.js";
export function generateJavaScript(
model: Model,
filePath: string,
destination: string | undefined
): string {
const data = extractDestinationAndName(filePath, destination);
const generatedFilePath = `${path.join(data.destination, data.name)}.js`;
const fileNode = new CompositeGeneratorNode();
fileNode.append('"use strict";', NL, NL);
if (!fs.existsSync(data.destination)) {
fs.mkdirSync(data.destination, { recursive: true });
}
fs.writeFileSync(generatedFilePath, toString(fileNode));
return generatedFilePath;
}

View file

@ -0,0 +1,42 @@
import type { Model } from '../language/generated/ast.js';
import chalk from 'chalk';
import { Command } from 'commander';
import { JsTransformLangLanguageMetaData } from '../language/generated/module.js';
import { createJsTransformLangServices } from '../language/js-transform-lang-module.js';
import { extractAstNode } from './cli-util.js';
import { generateJavaScript } from './generator.js';
import { NodeFileSystem } from 'langium/node';
import * as url from 'node:url';
import * as fs from 'node:fs/promises';
import * as path from 'node:path';
const __dirname = url.fileURLToPath(new URL('.', import.meta.url));
const packagePath = path.resolve(__dirname, '..', '..', 'package.json');
const packageContent = await fs.readFile(packagePath, 'utf-8');
export const generateAction = async (fileName: string, opts: GenerateOptions): Promise<void> => {
const services = createJsTransformLangServices(NodeFileSystem).JsTransformLang;
const model = await extractAstNode<Model>(fileName, services);
const generatedFilePath = generateJavaScript(model, fileName, opts.destination);
console.log(chalk.green(`JavaScript code generated successfully: ${generatedFilePath}`));
};
export type GenerateOptions = {
destination?: string;
}
export default function(): void {
const program = new Command();
program.version(JSON.parse(packageContent).version);
const fileExtensions = JsTransformLangLanguageMetaData.fileExtensions.join(', ');
program
.command('generate')
.argument('<file>', `source file (possible file extensions: ${fileExtensions})`)
.option('-d, --destination <dir>', 'destination directory of generating')
.description('generates JavaScript code that prints "Hello, {name}!" for each greeting in a source file')
.action(generateAction);
program.parse(process.argv);
}

View file

@ -0,0 +1,58 @@
import type { LanguageClientOptions, ServerOptions} from 'vscode-languageclient/node.js';
import * as vscode from 'vscode';
import * as path from 'node:path';
import { LanguageClient, TransportKind } from 'vscode-languageclient/node.js';
let client: LanguageClient;
// This function is called when the extension is activated.
export function activate(context: vscode.ExtensionContext): void {
client = startLanguageClient(context);
}
// This function is called when the extension is deactivated.
export function deactivate(): Thenable<void> | undefined {
if (client) {
return client.stop();
}
return undefined;
}
function startLanguageClient(context: vscode.ExtensionContext): LanguageClient {
const serverModule = context.asAbsolutePath(path.join('out', 'language', 'main.cjs'));
// The debug options for the server
// --inspect=6009: runs the server in Node's Inspector mode so VS Code can attach to the server for debugging.
// By setting `process.env.DEBUG_BREAK` to a truthy value, the language server will wait until a debugger is attached.
const debugOptions = { execArgv: ['--nolazy', `--inspect${process.env.DEBUG_BREAK ? '-brk' : ''}=${process.env.DEBUG_SOCKET || '6009'}`] };
// If the extension is launched in debug mode then the debug server options are used
// Otherwise the run options are used
const serverOptions: ServerOptions = {
run: { module: serverModule, transport: TransportKind.ipc },
debug: { module: serverModule, transport: TransportKind.ipc, options: debugOptions }
};
const fileSystemWatcher = vscode.workspace.createFileSystemWatcher('**/*.jstl');
context.subscriptions.push(fileSystemWatcher);
// Options to control the language client
const clientOptions: LanguageClientOptions = {
documentSelector: [{ scheme: 'file', language: 'js-transform-lang' }],
synchronize: {
// Notify the server about file changes to files contained in the workspace
fileEvents: fileSystemWatcher
}
};
// Create the language client and start the client.
const client = new LanguageClient(
'js-transform-lang',
'js-transform-lang',
serverOptions,
clientOptions
);
// Start the client. This will also launch the server
client.start();
return client;
}

View file

@ -0,0 +1,85 @@
import type {
DefaultSharedModuleContext,
LangiumServices,
LangiumSharedServices,
Module,
PartialLangiumServices,
} from "langium";
import {
createDefaultModule,
createDefaultSharedModule,
inject,
} from "langium";
import {
JsTransformLangGeneratedModule,
JsTransformLangGeneratedSharedModule,
} from "./generated/module.js";
import {
JsTransformLangValidator,
registerValidationChecks,
} from "./js-transform-lang-validator.js";
/**
* Declaration of custom services - add your own service classes here.
*/
export type JsTransformLangAddedServices = {
validation: {
JsTransformLangValidator: JsTransformLangValidator;
};
};
/**
* Union of Langium default services and your custom services - use this as constructor parameter
* of custom service classes.
*/
export type JsTransformLangServices = LangiumServices &
JsTransformLangAddedServices;
/**
* Dependency injection module that overrides Langium default services and contributes the
* declared custom services. The Langium defaults can be partially specified to override only
* selected services, while the custom services must be fully specified.
*/
export const JsTransformLangModule: Module<
JsTransformLangServices,
PartialLangiumServices & JsTransformLangAddedServices
> = {
validation: {
JsTransformLangValidator: () => new JsTransformLangValidator(),
},
};
/**
* Create the full set of services required by Langium.
*
* First inject the shared services by merging two modules:
* - Langium default shared services
* - Services generated by langium-cli
*
* Then inject the language-specific services by merging three modules:
* - Langium default language-specific services
* - Services generated by langium-cli
* - Services specified in this file
*
* @param context Optional module context with the LSP connection
* @returns An object wrapping the shared services and the language-specific services
*/
export function createJsTransformLangServices(
context: DefaultSharedModuleContext
): {
shared: LangiumSharedServices;
JsTransformLang: JsTransformLangServices;
} {
const shared = inject(
createDefaultSharedModule(context),
JsTransformLangGeneratedSharedModule
);
const JsTransformLang = inject(
createDefaultModule({ shared }),
JsTransformLangGeneratedModule,
JsTransformLangModule
);
shared.ServiceRegistry.register(JsTransformLang);
registerValidationChecks(JsTransformLang);
return { shared, JsTransformLang };
}

View file

@ -0,0 +1,34 @@
import type { ValidationAcceptor, ValidationChecks } from "langium";
import type { JsTransformLangAstType, Proposal } from "./generated/ast.js";
import type { JsTransformLangServices } from "./js-transform-lang-module.js";
/**
* Register custom validation checks.
*/
export function registerValidationChecks(services: JsTransformLangServices) {
const registry = services.validation.ValidationRegistry;
const validator = services.validation.JsTransformLangValidator;
const checks: ValidationChecks<JsTransformLangAstType> = {
Proposal: validator.checkPersonStartsWithCapital,
};
registry.register(checks, validator);
}
/**
* Implementation of custom validations.
*/
export class JsTransformLangValidator {
checkPersonStartsWithCapital(
proposal: Proposal,
accept: ValidationAcceptor
): void {
if (proposal.code) {
if (proposal.code === "") {
accept("warning", "You are running with empty code here", {
node: proposal,
property: "code",
});
}
}
}
}

View file

@ -0,0 +1,19 @@
grammar JsTransformLang
terminal PROPOSALNAME: /[(][_a-zA-Z][\w]*[)]/;
terminal STRING: /"(\\.|[^"\\])*"|'(\\.|[^'\\])*'/;
entry Model:
(proposals+=Proposal);
Proposal:
"proposal" proposalName=PROPOSALNAME "{"
"applicable" "to" "{"
code=STRING
"}"
"replace" "with" "{"
code=STRING
"}"
"}"
;

View file

@ -0,0 +1,13 @@
import { startLanguageServer } from 'langium';
import { NodeFileSystem } from 'langium/node';
import { createConnection, ProposedFeatures } from 'vscode-languageserver/node.js';
import { createJsTransformLangServices } from './js-transform-lang-module.js';
// Create a connection to the client
const connection = createConnection(ProposedFeatures.all);
// Inject the shared services and language-specific services
const { shared } = createJsTransformLangServices({ connection, ...NodeFileSystem });
// Start the language server with the shared services
startLanguageServer(shared);

View file

@ -0,0 +1,24 @@
{
"compilerOptions": {
"target": "ES2017",
"module": "Node16",
"lib": ["ESNext"],
"sourceMap": true,
"outDir": "out",
"strict": true,
"noUnusedLocals": true,
"noImplicitReturns": true,
"noImplicitOverride": true,
"moduleResolution": "Node16",
"esModuleInterop": true,
"skipLibCheck": true,
"forceConsistentCasingInFileNames": true
},
"include": [
"src/**/*.ts"
],
"exclude": [
"out",
"node_modules"
]
}

15
dsl_files/test.jstl Normal file
View file

@ -0,0 +1,15 @@
proposal (async) {
applicable to {
let _$_a_$_ = await _$_expr_$_();
console.log(_$_a_$_);
}
replace with {
_$_expr_$_().then(() => {
console.log(_$_a_$_);
})
}
}

View file

@ -1,19 +1,23 @@
{
"name": "didactic-chainsaw",
"module": "index.ts",
"type": "module",
"devDependencies": {
"@swc/cli": "^0.1.62",
"@types/node": "^20.5.9",
"bun-types": "latest",
"typescript": "^5.2.2"
},
"dependencies": {
"@babel/generator": "^7.23.0",
"@babel/parser": "^7.23.0",
"@babel/traverse": "^7.23.0",
"babel": "^6.23.0",
"bun": "^1.0.4",
"ts-node": "^10.9.1"
}
"name": "didactic-chainsaw",
"module": "index.ts",
"type": "module",
"scripts": {
"run": "bun run src/index.ts",
"watch": "bun --watch src/index.ts"
},
"devDependencies": {
"@swc/cli": "^0.1.62",
"@types/node": "^20.5.9",
"bun-types": "latest",
"typescript": "^5.2.2"
},
"dependencies": {
"@babel/generator": "^7.23.0",
"@babel/parser": "^7.23.0",
"@babel/traverse": "^7.23.0",
"babel": "^6.23.0",
"bun": "^1.0.4",
"ts-node": "^10.9.1"
}
}

View file

@ -1,30 +0,0 @@
import * as babelparser from "../babel/packages/babel-parser/lib";
import traverse from "../babel/packages/babel-traverse/lib";
import generate from "../babel/packages/babel-generator/lib";
const main = () => {
let code_To_Insert = "697 + 457";
let code = "1 + 1;";
let ast = babelparser.parse(code);
console.log(ast);
let insert_ast = babelparser.parse(code);
traverse(ast, {
enter(path) {
if (path.isBinaryExpression({ operator: "+" })) {
path.node.operator = "@@@";
}
},
});
console.log(JSON.stringify(ast, null, 4));
const out = generate(ast, {}, code);
console.log("input: " + code);
console.log("output: " + out.code);
let inout = babelparser.parse(out.code);
console.log(inout);
console.log(generate(inout, {}, code));
};
main();

5
src/index.ts Normal file
View file

@ -0,0 +1,5 @@
import * as babelparser from "../babel/packages/babel-parser";
const main = (): void => {};
main();

2
src/placeholders.js Normal file
View file

@ -0,0 +1,2 @@

3
src/structurizer.js Normal file
View file

@ -0,0 +1,3 @@

View file

@ -1 +0,0 @@
var oldVariableName = "Old_value";

View file

@ -1,13 +1,14 @@
{
"compilerOptions": {
"rootDir": "src",
"outDir": "dist",
"strict": true,
"target": "es6",
"module": "commonjs",
"sourceMap": true,
"esModuleInterop": true,
"moduleResolution": "node",
"types": ["bun-types"]
}
}
"compilerOptions": {
"rootDir": "src",
"outDir": "dist",
"strict": true,
"target": "es6",
"module": "commonjs",
"sourceMap": true,
"esModuleInterop": true,
"moduleResolution": "node",
"types": ["bun-types"]
},
"include": ["src", "babel/packages/babel-parser"]
}