dietc.complete source code

1 module dietc.complete;
2 
3 import dietc.lexer;
4 import dietc.parser;
5 
6 import std.algorithm;
7 import std.array;
8 import std.string;
9 import std.uni;
10 import std.meta;
11 
12 /// Delegate to provide other files that are being extended
13 alias FileProvider = DietInput delegate(string name);
14 
15 enum CompletionType
16 {
17 	none,
18 	tag,
19 	attribute,
20 	value,
21 	reference,
22 	cssName,
23 	cssValue,
24 	d,
25 	meta
26 }
27 
28 struct Completion
29 {
30 	CompletionType type;
31 	string text;
32 	string definition;
33 	string documentation;
34 
35 	string referenceFile;
36 	size_t[2] referenceRange;
37 	bool preselected;
38 
39 	auto preselect()
40 	{
41 		preselected = true;
42 		return this;
43 	}
44 
45 	static immutable const(Completion)[] completeD = [Completion(CompletionType.d, "", "<d source>")];
46 }
47 
48 struct TagInfo
49 {
50 	struct Attribute
51 	{
52 		string name;
53 		CompletionSource completion;
54 	}
55 
56 	string tag;
57 	Attribute[] attributes;
58 }
59 
60 private const(TagInfo)[] parseTagInfos(string info)
61 {
62 	auto ret = appender!(const(TagInfo)[]);
63 	auto attributeBase = appender!(TagInfo.Attribute[]);
64 	CompletionSource[string] enumCompletions;
65 
66 	auto parseAttribute(string attr)
67 	{
68 		auto colon = attr.indexOf(":");
69 		if (colon == -1)
70 			throw new Exception("Malformed attribute: " ~ attr);
71 		TagInfo.Attribute retAttr;
72 		retAttr.name = attr[0 .. colon];
73 		auto value = attr[colon + 1 .. $];
74 		auto dot = value.indexOf('.');
75 
76 		if (auto exist = value in enumCompletions)
77 			retAttr.completion = *exist;
78 		else if (dot != -1)
79 			retAttr.completion = new AttributeValueByTagNameComplete(value[0 .. dot], value[dot + 1 .. $]);
80 		else
81 			throw new Exception("Unknown attribute value " ~ value);
82 
83 		return cast(const) retAttr;
84 	}
85 
86 	foreach (line; info.lineSplitter)
87 	{
88 		line = line.strip;
89 		if (line.startsWith("//") || !line.length)
90 			continue;
91 		if (line.startsWith("e "))
92 		{
93 			line = line[1 .. $].stripLeft;
94 			auto larr = line.indexOf("<");
95 			if (larr == -1)
96 				throw new Exception("Malformed enum line: " ~ line);
97 			auto name = line[0 .. larr].stripRight;
98 			auto values = line[larr + 1 .. $].stripLeft.splitter;
99 			enumCompletions[name] = new EnumComplete(values.map!(a => Completion(CompletionType.value, a)).array);
100 		}
101 		else if (line.startsWith("t "))
102 		{
103 			line = line[1 .. $].stripLeft;
104 			auto larr = line.indexOf("<");
105 			if (larr == -1)
106 				throw new Exception("Malformed enum line: " ~ line);
107 			auto name = line[0 .. larr].stripRight;
108 			auto attrs = appender(attributeBase.data);
109 			foreach (attr; line[larr + 1 .. $].stripLeft.splitter)
110 				attrs ~= parseAttribute(attr);
111 			ret ~= const TagInfo(name, attrs.data);
112 		}
113 		else if (line.startsWith("tbase "))
114 		{
115 			line = line["tbase".length .. $].stripLeft;
116 			attributeBase.clear();
117 			foreach (attr; line.splitter)
118 				attributeBase ~= parseAttribute(attr);
119 		}
120 		else throw new Exception("Invalid line " ~ line);
121 	}
122 	return ret.data;
123 }
124 
125 __gshared const(TagInfo)[] tagInfos;
126 shared static this()
127 {
128 	tagInfos = import("html.txt").parseTagInfos;
129 }
130 
131 static immutable Completion[] tagCompletions = import("tags.txt").strip
132 	.splitLines.map!(a => Completion(CompletionType.tag, a)).array;
133 
134 //dfmt off
135 // https://github.com/rejectedsoftware/diet-ng/blob/f65a31def40f40cba2bf03a8f2093821e28a26d3/source/diet/html.d#L428
136 static immutable Completion[] doctypeCompletions = [
137 	Completion(CompletionType.value, "html", `<!DOCTYPE html>`).preselect,
138 	Completion(CompletionType.value, "xml", `<?xml version="1.0" encoding="utf-8" ?>`),
139 	Completion(CompletionType.value, "transitional", `<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">`),
140 	Completion(CompletionType.value, "strict", `<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd">`),
141 	Completion(CompletionType.value, "frameset", `<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Frameset//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-frameset.dtd">`),
142 	Completion(CompletionType.value, "1.1", `<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.1//EN" "http://www.w3.org/TR/xhtml11/DTD/xhtml11.dtd">`),
143 	Completion(CompletionType.value, "basic", `<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML Basic 1.1//EN" "http://www.w3.org/TR/xhtml-basic/xhtml-basic11.dtd">`),
144 	Completion(CompletionType.value, "mobile", `<!DOCTYPE html PUBLIC "-//WAPFORUM//DTD XHTML Mobile 1.2//EN" "http://www.openmobilealliance.org/tech/DTD/xhtml-mobile12.dtd">`),
145 ];
146 //dfmt on
147 
148 interface CompletionSource
149 {
150 	const(Completion)[] complete(string identifier, AST[] context, DietComplete engine, size_t offset);
151 }
152 
153 class EnumComplete : CompletionSource
154 {
155 	const Completion[] available;
156 
157 	this(const Completion[] available)
158 	{
159 		this.available = available;
160 	}
161 
162 	const(Completion)[] complete(string identifier, AST[], DietComplete, size_t) const
163 	{
164 		return available.filter!(a => a.text.asLowerCase.startsWith(identifier.asLowerCase)).array;
165 	}
166 }
167 
168 /// Finds attribute values from other tags based on a tag name such as "find id values from table elements".
169 class AttributeValueByTagNameComplete : CompletionSource
170 {
171 	string attribute, tag;
172 
173 	this(string attribute, string tag)
174 	{
175 		this.attribute = attribute;
176 		this.tag = tag;
177 	}
178 
179 	const(Completion)[] complete(string identifier, AST[] context, DietComplete, size_t) const
180 	{
181 		return null;
182 	}
183 }
184 
185 class AttributeNameComplete : CompletionSource
186 {
187 	const TagInfo[] tagInfos;
188 
189 	this(in TagInfo[] tagInfos)
190 	{
191 		this.tagInfos = tagInfos;
192 	}
193 
194 	const(Completion)[] complete(string identifier, AST[] context, DietComplete, size_t) const
195 	{
196 		TagNode tag;
197 		foreach_reverse (node; context)
198 			if (cast(TagNode) node)
199 			{
200 				tag = cast(TagNode) node;
201 				break;
202 			}
203 
204 		if (tag)
205 		{
206 			foreach (info; tagInfos)
207 			{
208 				if (sicmp(info.tag, tag.name) == 0)
209 				{
210 					const(Completion)[] completion;
211 					foreach (attr; info.attributes)
212 						if (attr.name.startsWith(identifier.asLowerCase))
213 							completion ~= Completion(CompletionType.attribute, attr.name);
214 					return completion;
215 				}
216 			}
217 		}
218 		return null;
219 	}
220 }
221 
222 class AttributeValueComplete : CompletionSource
223 {
224 	const TagInfo[] tagInfos;
225 
226 	this(in TagInfo[] tagInfos)
227 	{
228 		this.tagInfos = tagInfos;
229 	}
230 
231 	const(Completion)[] complete(string identifier, AST[] context, DietComplete engine, size_t offset) const
232 	{
233 		TagNode tag;
234 		TagNode.AttributeAST attribute;
235 		foreach_reverse (node; context)
236 		{
237 			if (cast(TagNode.AttributeAST) node)
238 				attribute = cast(TagNode.AttributeAST) node;
239 
240 			if (cast(TagNode) node)
241 			{
242 				tag = cast(TagNode) node;
243 				break;
244 			}
245 		}
246 
247 		identifier = identifier.reduceToLastIdentifier;
248 
249 		if (tag)
250 		{
251 			foreach (info; tagInfos)
252 			{
253 				if (sicmp(info.tag, tag.name) == 0)
254 				{
255 					Completion[] completion;
256 					foreach (attr; info.attributes)
257 						if (sicmp(attr.name, attribute.name) == 0)
258 							completion ~= (cast()attr.completion).complete(identifier, context, engine, offset);
259 					return cast(const(Completion)[]) completion.sort!"a.text < b.text".uniq!"a.text == b.text".array;
260 				}
261 			}
262 		}
263 		return null;
264 	}
265 }
266 
267 class DietComplete
268 {
269 	FileProvider provider;
270 	ASTParser parser;
271 	EnumComplete tags;
272 	EnumComplete doctypes;
273 	AttributeNameComplete attributeNames;
274 	AttributeValueComplete attributeValues;
275 
276 	this(string file)
277 	{
278 		import std.path : dirName;
279 		this(DietInput.fromFile(file), defaultFileProvider(dirName(file)));
280 	}
281 
282 	this(DietInput root, FileProvider provider)
283 	{
284 		tags = new EnumComplete(tagCompletions);
285 		doctypes = new EnumComplete(doctypeCompletions);
286 		attributeNames = new AttributeNameComplete(tagInfos);
287 		attributeValues = new AttributeValueComplete(tagInfos);
288 
289 		this.provider = provider;
290 		root.reset();
291 		parser = ASTParser(root);
292 		parser.parseDocument();
293 	}
294 
295 	void reparse(string content)
296 	{
297 		parser.input.code = content;
298 		parser.input.reset();
299 		parser.parseDocument();
300 	}
301 
302 	static FileProvider defaultFileProvider(string dir)
303 	{
304 		return (name) {
305 			import std.file : exists;
306 			import std.path : chainPath, withExtension;
307 
308 			if (exists(chainPath(dir, name.withExtension(".dt"))))
309 				return DietInput.fromFile(chainPath(dir, name.withExtension(".dt")));
310 			else
311 				return DietInput.init;
312 		};
313 	}
314 
315 	const(Completion)[] completeAt(size_t offset)
316 	{
317 		auto tree = parser.searchAST(offset);
318 
319 		auto contentLess = tree;
320 		if (cast(Document) contentLess[0])
321 			while (cast(TextLine) contentLess[$ - 1] || cast(TextLine.PartAST) contentLess[$ - 1])
322 				contentLess.length--;
323 		assert(contentLess.length >= 1);
324 
325 		if (cast(DStatement) tree[$ - 1] || cast(Assignment) tree[$ - 1] || cast(RawAssignment) tree[$ - 1])
326 		{
327 			auto stmt = cast(IStringContainer) tree[$ - 1];
328 			if (offset.withinRange([stmt.token.range[0] + 1, stmt.token.range[0] + 1 + stmt.content.length]))
329 				return Completion.completeD;
330 			else
331 				return null; // before "-" character or in newline at end
332 		}
333 
334 		if (auto expr = cast(Expression) tree[$ - 1])
335 		{
336 			auto code = expr.content;
337 			if (!code.all!isNumber && !code.isPlainString)
338 				return Completion.completeD;
339 		}
340 
341 		if (tree.length >= 2)
342 		{
343 			if (auto expr = cast(Expression) tree[$ - 1])
344 				if (auto attr = cast(TagNode.AttributeAST) tree[$ - 2])
345 				{
346 					string exprCode = parser.input.read([expr.token.range[0], offset]);
347 					return attributeValues.complete(exprCode, tree, this, offset);
348 				}
349 		}
350 
351 		if (tree.length >= 3)
352 		{
353 			if (auto part = cast(TextLine.PartAST) tree[$ - 1])
354 				if (auto line = cast(TextLine) tree[$ - 2])
355 					if (auto tag = cast(TagNode) tree[$ - 3])
356 						if (line._parts.length == 1 && part.part.raw.length)
357 						{
358 							string text = part.part.raw[0 .. offset - part.token.range[0]];
359 							if (tag.name == "doctype")
360 								return doctypes.complete(text, tree, this, offset);
361 						}
362 		}
363 
364 		if (auto tag = cast(TagNode) contentLess[$ - 1])
365 		{
366 			if (offset.withinRange(tag.tag.range))
367 			{
368 				string written = parser.input.read([tag.tag.range[0], offset]);
369 				auto ret = tags.complete(written, contentLess, this, offset);
370 				if (contentLess.length == 2) // top level element
371 				{
372 					if ("doctype".startsWith(written.asLowerCase))
373 						ret ~= Completion(CompletionType.meta, "doctype").preselect;
374 					if ("html".startsWith(written.asLowerCase))
375 						ret ~= Completion(CompletionType.tag, "html").preselect;
376 				}
377 				else if (contentLess.length > 2)
378 				{
379 					if (auto parent = cast(TagNode) contentLess[$ - 2])
380 					{
381 						if (parent.name == "html")
382 						{
383 							if ("head".startsWith(written.asLowerCase))
384 								ret ~= Completion(CompletionType.tag, "head").preselect;
385 							if ("body".startsWith(written.asLowerCase))
386 								ret ~= Completion(CompletionType.tag, "body").preselect;
387 						}
388 					}
389 				}
390 				return ret;
391 			}
392 			else if (offset.withinRange(tag.attributesRange)) // somewhere random in attributes but not in name or value
393 				return attributeNames.complete("", contentLess, this, offset);
394 		}
395 
396 		if (contentLess.length >= 2)
397 		{
398 			if (auto tag = cast(TagNode) contentLess[$ - 2])
399 				if (auto attr = cast(TagNode.AttributeAST) contentLess[$ - 1])
400 					if (offset.withinRange(attr.token.range))
401 						return attributeNames.complete(parser.input.read([attr.token.range[0], offset]), contentLess, this, offset);
402 
403 			if (auto text = cast(TextLine) contentLess[$ - 1]) // must be an empty textline because otherwise a PartAST would be here
404 				if (auto tag = cast(TagNode) contentLess[$ - 2])
405 				{
406 					if (tag.name == "doctype")
407 						return doctypes.complete("", contentLess, this, offset);
408 				}
409 		}
410 
411 		if (tree.length == 1)
412 		{
413 			auto ret = tags.complete("", tree, this, offset);
414 			if (cast(Document)tree[0] && (cast(Document)tree[0])._children.length == 0)
415 				ret ~= Completion(CompletionType.meta, "doctype").preselect;
416 			return ret;
417 		}
418 
419 		return null;
420 	}
421 }
422 
423 void extractD(DietComplete complete, size_t offset, out string code, out size_t codeOffset, string prefix = null)
424 {
425 	return complete.parser.root.extractD(offset, code, codeOffset, prefix);
426 }
427 
428 void extractD(AST root, size_t offset, out string code, out size_t codeOffset, string prefix = null)
429 {
430 	code = prefix;
431 	codeOffset = size_t.max;
432 	class CodeVisitorImpl : ASTVisitor
433 	{
434 		override void visit(DStatement stmt) in (stmt !is null)
435 		{
436 			if (offset.withinRange(stmt.token.range))
437 				codeOffset = code.length + (offset - (stmt.token.range[0] + stmt.token.content.length));
438 			code ~= stmt.content;
439 			if (stmt.children.length)
440 			{
441 				code ~= "{/*children*/";
442 				stmt.accept(this);
443 				code ~= "}";
444 			}
445 		}
446 
447 		static foreach (T; AliasSeq!(Expression, Assignment, RawAssignment))
448 			override void visit(T expr) in (expr !is null)
449 			{
450 				static if (is(T == Expression))
451 					enum typeOffset = 0;
452 				else static if (is(T == Assignment))
453 					enum typeOffset = 1; // offset the equal of tag=
454 				else static if (is(T == RawAssignment))
455 					enum typeOffset = 2; // offset the bang-equal of tag!=
456 
457 				code ~= "__diet_value(";
458 				if (offset.withinRange(expr.token.range))
459 					codeOffset = code.length + (offset - typeOffset - expr.token.range[0]);
460 				code ~= expr.content;
461 				code ~= ");";
462 			}
463 
464 		override void visit(Comment comment) in (comment !is null)
465 		{
466 		}
467 
468 		override void visit(HiddenComment comment) in (comment !is null)
469 		{
470 		}
471 
472 		override void visit(Document doc) in (doc !is null)
473 		{
474 			code ~= "void __diet_document() {";
475 			doc.accept(this);
476 			code ~= "}";
477 		}
478 
479 		static foreach (T; AliasSeq!(TagNode, TagNode.AttributeAST))
480 			override void visit(T ast) in (ast !is null)
481 			{
482 				code ~= "{";
483 				ast.accept(this);
484 				code ~= "}";
485 			}
486 
487 		alias visit = ASTVisitor.visit;
488 	}
489 
490 	new CodeVisitorImpl().visit(root);
491 }
492 
493 bool isPlainString(string code)
494 {
495 	if (!code.length)
496 		return false;
497 	auto quote = code[0];
498 	if (quote != '\'' && quote != '"' && quote != '`')
499 		return false;
500 	if (code[$ - 1] != quote)
501 		return false;
502 	// TODO: better string check
503 	return true;
504 }
505 
506 string reduceToLastIdentifier(string identifier)
507 {
508 	foreach_reverse (i, c; identifier)
509 	{
510 		if (c.isNumber || (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || c == '_' || c == '-')
511 			continue;
512 		return identifier[i + 1 .. $];
513 	}
514 	return identifier;
515 }
516 
517 unittest
518 {
519 	import std.conv;
520 
521 	const(Completion)[] testComplete(string text, size_t at,
522 			string file = __FILE__ ~ ":" ~ __LINE__.to!string)
523 	{
524 		DietInput input;
525 		input.file = file;
526 		input.code = text;
527 		return new DietComplete(input, cast(FileProvider)(name) {
528 			assert(false, "Can't import " ~ name ~ " in test");
529 		}).completeAt(at);
530 	}
531 
532 	assert(testComplete("a", 0).canFind!(a => a.text == "textarea"));
533 	assert(testComplete("t", 1).canFind!(a => a.text == "textarea"));
534 	assert(testComplete("", 0).canFind!(a => a.text == "textarea"));
535 	assert(testComplete("div\n\t", 5).canFind!(a => a.text == "textarea"));
536 }
537 
538 unittest
539 {
540 	import std.conv;
541 
542 	DietInput input;
543 	input.file = "stdin";
544 	input.code = `foo
545 	- int item = 3;
546 	p #{item.foobar} bar
547 	a(attr=foo.bar)= foo.bar
548 `;
549 	auto c = new DietComplete(input, cast(FileProvider)(name) {
550 		assert(false, "Can't import " ~ name ~ " in test");
551 	});
552 
553 	assert(c.completeAt(23).canFind!(a => a.text == "pre"));
554 	assert(c.completeAt(39).length == 0);
555 	assert(c.completeAt(38).length == 0);
556 	assert(c.completeAt(24).length == 0);
557 	assert(c.completeAt(25).length == 0);
558 
559 	void checkDBounds(size_t start, size_t end)
560 	{
561 		assert(c.completeAt(start - 1) !is Completion.completeD);
562 		assert(c.completeAt(start) is Completion.completeD);
563 		assert(c.completeAt(end) is Completion.completeD);
564 		assert(c.completeAt(end + 1) !is Completion.completeD);
565 	}
566 
567 	checkDBounds(6, 20);
568 	checkDBounds(26, 37);
569 	checkDBounds(51, 58);
570 	checkDBounds(60, 68);
571 
572 	string code;
573 	size_t offset;
574 
575 	c.extractD(28, code, offset);
576 	assert(code == `void __diet_document() {{ int item = 3;{__diet_value(item.foobar);}{{__diet_value(foo.bar);}__diet_value( foo.bar);}}}`);
577 	assert(offset == 55);
578 
579 	c.extractD(37, code, offset);
580 	assert(offset == 64);
581 
582 	c.extractD(38, code, offset);
583 	assert(offset == size_t.max);
584 
585 	c.extractD(25, code, offset);
586 	assert(offset == size_t.max);
587 
588 	c.extractD(7, code, offset);
589 	assert(offset == 26);
590 
591 	c.extractD(20, code, offset);
592 	assert(offset == 39);
593 
594 	c.extractD(51, code, offset);
595 	assert(offset == 82);
596 
597 	c.extractD(58, code, offset);
598 	assert(offset == 89);
599 
600 	c.extractD(61, code, offset);
601 	assert(offset == 106);
602 
603 	c.extractD(68, code, offset);
604 	assert(offset == 113);
605 }