I was trying to implement this using a different language. but it doesn't seem to handle the character that has been inserted, This actually works with english I tweaked it in amharic, and I was thinking to add more things if the handling works like creating an editor or something. If anyone is willing to help, I would appreciate it. I know that I should normalize the unicode characters, but I don't know efficient way I could do that.
function InputStream(input) {
var pos = 0, line = 1, col = 0;
return {
next : next,
peek : peek,
eof : eof,
croak : croak,
};
function next() {
var ch = input.charAt(pos++);
if (ch == "\n") line++, col = 0; else col++;
return ch;
}
function peek() {
return input.charAt(pos);
}
function eof() {
return peek() == "";
}
function croak(msg) {
throw new Error(msg + " (" + line + ":" + col + ")");
}
}
function TokenStream(input) {
var current = null;
var keywords = " ከ ወደ ሌላ ስራ ስራ አውነት ሀሰት ";
return {
next : next,
peek : peek,
eof : eof,
croak : input.croak
};
function is_keyword(x) {
return keywords.indexOf(" " + x + " ") >= 0;
}
function is_digit(ch) {
return /[0-9]/i.test(ch);
}
function is_id_start(ch) {
return /[^\u1380-\u1380f]+[a-z_]/i.test(ch);
}
function is_id(ch) {
return is_id_start(ch) || "?!-<>=0123456789".indexOf(ch) >= 0;
}
function is_op_char(ch) {
return "+-*/%=&|<>!".indexOf(ch) >= 0;
}
function is_punc(ch) {
return "፣፤(){}[]".indexOf(ch) >= 0;
}
function is_whitespace(ch) {
return " \t\n".indexOf(ch) >= 0;
}
function read_while(predicate) {
var str = "";
while (!input.eof() && predicate(input.peek()))
str += input.next();
return str;
}
function read_number() {
var has_dot = false;
var number = read_while(function(ch){
if (ch == ".") {
if (has_dot) return false;
has_dot = true;
return true;
}
return is_digit(ch);
});
return { type: "num", value: parseFloat(number) };
}
function read_ident() {
var id = read_while(is_id);
return {
type : is_keyword(id) ? "kw" : "var",
value : id
};
}
function read_escaped(end) {
var escaped = false, str = "";
input.next();
while (!input.eof()) {
var ch = input.next();
if (escaped) {
str += ch;
escaped = false;
} else if (ch == "\\") {
escaped = true;
} else if (ch == end) {
break;
} else {
str += ch;
}
}
return str;
}
function read_string() {
return { type: "str", value: read_escaped('"') };
}
function skip_comment() {
read_while(function(ch){ return ch != "\n" });
input.next();
}
function read_next() {
read_while(is_whitespace);
if (input.eof()) return null;
var ch = input.peek();
if (ch == "#") {
skip_comment();
return read_next();
}
if (ch == '"') return read_string();
if (is_digit(ch)) return read_number();
if (is_id_start(ch)) return read_ident();
if (is_punc(ch)) return {
type : "punc",
value : input.next()
};
if (is_op_char(ch)) return {
type : "op",
value : read_while(is_op_char)
};
input.croak("Can't handle character: " + ch);
}
function peek() {
return current || (current = read_next());
}
function next() {
var tok = current;
current = null;
return tok || read_next();
}
function eof() {
return peek() == null;
}
}
function parse(input) {
var PRECEDENCE = {
"=": 1,
"||": 2,
"&&": 3,
"<": 7, ">": 7, "<=": 7, ">=": 7, "==": 7, "!=": 7,
"+": 10, "-": 10,
"*": 20, "/": 20, "%": 20,
};
var FALSE = { type: "bool", value: false };
return parse_toplevel();
function is_punc(ch) {
var tok = input.peek();
return tok && tok.type == "punc" && (!ch || tok.value == ch) && tok;
}
function is_kw(kw) {
var tok = input.peek();
return tok && tok.type == "kw" && (!kw || tok.value == kw) && tok;
}
function is_op(op) {
var tok = input.peek();
return tok && tok.type == "op" && (!op || tok.value == op) && tok;
}
function skip_punc(ch) {
if (is_punc(ch)) input.next();
else input.croak("Expecting punctuation: \"" + ch + "\"");
}
function skip_kw(kw) {
if (is_kw(kw)) input.next();
else input.croak("Expecting keyword: \"" + kw + "\"");
}
function skip_op(op) {
if (is_op(op)) input.next();
else input.croak("Expecting operator: \"" + op + "\"");
}
function unexpected() {
input.croak("Unexpected token: " + JSON.stringify(input.peek()));
}
function maybe_binary(left, my_prec) {
var tok = is_op();
if (tok) {
var his_prec = PRECEDENCE[tok.value];
if (his_prec > my_prec) {
input.next();
return maybe_binary({
type : tok.value == "=" ? "assign" : "binary",
operator : tok.value,
left : left,
right : maybe_binary(parse_atom(), his_prec)
}, my_prec);
}
}
return left;
}
function delimited(start, stop, separator, parser) {
var a = [], first = true;
skip_punc(start);
while (!input.eof()) {
if (is_punc(stop)) break;
if (first) first = false; else skip_punc(separator);
if (is_punc(stop)) break;
a.push(parser());
}
skip_punc(stop);
return a;
}
function parse_call(func) {
return {
type: "call",
func: func,
args: delimited("(", ")", "፣", parse_expression),
};
}
function parse_varname() {
var name = input.next();
if (name.type != "var") input.croak("Expecting variable name");
return name.value;
}
function parse_if() {
skip_kw("ከ");
var cond = parse_expression();
if (!is_punc("{")) skip_kw("ወደ");
var then = parse_expression();
var ret = {
type: "if",
cond: cond,
then: then,
};
if (is_kw("ሌላ")) {
input.next();
ret.else = parse_expression();
}
return ret;
}
function parse_lambda() {
return {
type: "lambda",
vars: delimited("(", ")", "፣", parse_varname),
body: parse_expression()
};
}
function parse_bool() {
return {
type : "bool",
value : input.next().value == "አውነት"
};
}
function maybe_call(expr) {
expr = expr();
return is_punc("(") ? parse_call(expr) : expr;
}
function parse_atom() {
return maybe_call(function(){
if (is_punc("(")) {
input.next();
var exp = parse_expression();
skip_punc(")");
return exp;
}
if (is_punc("{")) return parse_prog();
if (is_kw("if")) return parse_if();
if (is_kw("አውነት") || is_kw("ሀሰት")) return parse_bool();
if (is_kw("ስራ") || is_kw("ሥራ")) {
input.next();
return parse_lambda();
}
var tok = input.next();
if (tok.type == "var" || tok.type == "num" || tok.type == "str")
return tok;
unexpected();
});
}
function parse_toplevel() {
var prog = [];
while (!input.eof()) {
prog.push(parse_expression());
if (!input.eof()) skip_punc("፤");
}
return { type: "prog", prog: prog };
}
function parse_prog() {
var prog = delimited("{", "}", "፤", parse_expression);
if (prog.length == 0) return FALSE;
if (prog.length == 1) return prog[0];
return { type: "prog", prog: prog };
}
function parse_expression() {
return maybe_call(function(){
return maybe_binary(parse_atom(), 0);
});
}
}
function Environment(parent) {
this.vars = Object.create(parent ? parent.vars : null);
this.parent = parent;
}
Environment.prototype = {
extend: function() {
return new Environment(this);
},
lookup: function(name) {
var scope = this;
while (scope) {
if (Object.prototype.hasOwnProperty.call(scope.vars, name))
return scope;
scope = scope.parent;
}
},
get: function(name) {
if (name in this.vars)
return this.vars[name];
throw new Error("Undefined variable " + name);
},
set: function(name, value) {
var scope = this.lookup(name);
if (!scope && this.parent)
throw new Error("Undefined variable " + name);
return (scope || this).vars[name] = value;
},
def: function(name, value) {
return this.vars[name] = value;
}
};
function evaluate(exp, env) {
switch (exp.type) {
case "num":
case "str":
case "bool":
return exp.value;
case "var":
return env.get(exp.value);
case "assign":
if (exp.left.type != "ተጠቀመም")
throw new Error("Cannot assign to " + JSON.stringify(exp.left));
return env.set(exp.left.value, evaluate(exp.right, env));
case "binary":
return apply_op(exp.operator,
evaluate(exp.left, env),
evaluate(exp.right, env));
case "lambda":
return make_lambda(env, exp);
case "if":
var cond = evaluate(exp.cond, env);
if (cond !== false) return evaluate(exp.then, env);
return exp.else ? evaluate(exp.else, env) : false;
case "prog":
var val = false;
exp.prog.forEach(function(exp){ val = evaluate(exp, env) });
return val;
case "call":
var func = evaluate(exp.func, env);
return func.apply(null, exp.args.map(function(arg){
return evaluate(arg, env);
}));
default:
throw new Error("I don't know how to evaluate " + exp.type);
}
}
function apply_op(op, a, b) {
function num(x) {
if (typeof x != "number")
throw new Error("Expected number but got " + x);
return x;
}
function div(x) {
if (num(x) == 0)
throw new Error("Divide by zero");
return x;
}
switch (op) {
case "+": return num(a) + num(b);
case "-": return num(a) - num(b);
case "*": return num(a) * num(b);
case "/": return num(a) / div(b);
case "%": return num(a) % div(b);
case "&&": return a !== false && b;
case "||": return a !== false ? a : b;
case "<": return num(a) < num(b);
case ">": return num(a) > num(b);
case "<=": return num(a) <= num(b);
case ">=": return num(a) >= num(b);
case "==": return a === b;
case "!=": return a !== b;
}
throw new Error("Can't apply operator " + op);
}
function make_lambda(env, exp) {
function lambda() {
var names = exp.vars;
var scope = env.extend();
for (var i = 0; i < names.length; ++i)
scope.def(names[i], i < arguments.length ? arguments[i] : false);
return evaluate(exp.body, scope);
}
return lambda;
}
/* -----[ entry point for NodeJS ]----- */
var globalEnv = new Environment();
globalEnv.def("time", function(func){
try {
console.time("time");
return func();
} finally {
console.timeEnd("time");
}
});
if (typeof process != "undefined") (function(){
var util = require("util");
globalEnv.def("ፃፍ", function(val){
console.log(val);
});
globalEnv.def("print", function(val){
util.print(val);
});
var code = "";
process.stdin.setEncoding("utf8");
process.stdin.on("readable", function(){
var chunk = process.stdin.read();
if (chunk) code += chunk;
});
process.stdin.on("end", function(){
var ast = parse(TokenStream(InputStream(code)));
evaluate(ast, globalEnv);
});
})();
Here's the english version this should be how print works "println("hello world!");" and i would like to use "ፃፍ("ሰላም ሰላም")፤" where 'print' and ';' are converted to 'ፃፍ' and "፤
That's a lot of code, and no explanation of what it fails to do, other than "it doesn't seem to handle the character that has been inserted" which could mean just about anything. If I had an Amharic keyboard layout and knew how to type with it, I might give the code a try to see what's wrong, but I don't have one and I wouldn't know how to use it, so I didn't try.
However, I think your is_id and is_id_start function is not likely to work. You have:
function is_id_start(ch) {
return /[^\u1380-\u1380f]+[a-z_]/i.test(ch);
}
function is_id(ch) {
return is_id_start(ch) || "?!-<>=0123456789".indexOf(ch) >= 0;
}
I was a little puzzled to see !-<>= as possible identifier characters, since they are also operator characters. Presumably, the intention is that if an identifier is followed by one of those operators, there must be whitespace in between. But I don't think that's the problem you are experiencing with Amharic. That seems more likely to have to do with this slightly odd regular expression: /[^\u1380-\u1380f]+[a-z_]/i.
To start with, \u1380f is not a single unicode escape character. \u must be followed by exactly four hex digits, so the f is not part of the escape. It's just an ordinary f. That makes the range \u1380-\u1380, which consists of the single character ᎀ.
It's possible that you meant \u1380-\u138f, which would be 16 of the 453 Ethiopic letters. My knowledge of Amharic is pretty limited, and certainly not adequate to understand what differentiates those particular characters, so I can't even begin to guess whether that's reasonable.
However, you use that range in an inverted regular expression; even making that correction, what [^\u1380-\u138f] matches is any character other than a character in that range. That would include characters from all over the Unicode galaxy, including lots of other scripts, so I'm pretty certain that it is not what you intended.
Furthermore, your ID start pattern is actually /[^\u1380-\u1380f]+[a-z_]/i, which means "one or more [...] followed by a Latin alphabetic character or underscore". In other words, the pattern requires at least two characters to match, first an Amharic character (or, as written, anything other than one of those Amharic characters) and then a Latin alphabetic character.
That's clearly wrong, because you are matching that pattern against the result of input.peek(), which can only be a single character. So is_id is guaranteed to return false, which is probably related to your problem.
Personally, I think you'd be a lot better off using Unicode Property classes than trying to write down a list of valid identifier characters. (Why restrict identifiers to just English and Amharic, for example?) Ecmascript conveniently comes with the standard character sets recommended by the Unicode technical experts to be used for identifiers. You could just use them:
function is_id_start(ch) {
return /\p{XID_Start}/u.test(ch);
}
function is_id(ch) {
return /[\p{XID_Continue}?!-<>=]/u.test(ch);
}
Note that you must use the u flag to enable the use of Unicode properties.
If you really wanted to restrict that to Amharic and Latin, you could do so by also requiring that the characters also match [\p{Script=Ethiopic}\p{Script=Latin}]. (You can use a lookahead assertion to do that.)