[issue #759] delombok now plays nicely with exotic characters, encodings, and backslash U escapes.

doc/changelog.markdown

@@ -3,6 +3,7 @@ Lombok Changelog
 
 ### v1.14.9.shadow "<del>Edgy</del> Shadowy Guinea Pig"
 * Added a launcher to the lombok boot process which removes the need for `-Xbootclasspath` to be in your `eclipse.ini` file, and removes all non-public API and third party dependencies (such as ASM) from the lombok jar, thus removing them from your IDE's auto complete offerings in any project that uses lombok. For those debugging lombok, the launcher enables hot code replace which makes debugging a lot easier, as previously one was required to shut down the IDE, rebuild the jar, and relaunch. Add `-Dshadow.override.lombok=/path/to/lombok/bin` to the launch target for hot code replace.
+* BUGFIX: delombok: Using exotic characters in your source files would overzealously backslash-u escape them. Now, all characters are printed unescaped, assuming your chosen encoding can support them. Otherwise, they are escaped. [Issue #759](https://code.google.com/p/projectlombok/issues/detail?id=759)
 
 ### v1.14.8 (September 15th, 2014)
 * PERFORMANCE: The configuration system typically hit the filesystem twice per read configuration key instead of hardly ever. This is a continuation of [Issue #682](https://code.google.com/p/projectlombok/issues/detail?id=682).

src/delombok/lombok/delombok/Delombok.java

@@ -509,7 +509,7 @@ public class Delombok {
 			DelombokResult result = new DelombokResult(catcher.getComments(unit), unit, force || options.isChanged(unit), fps);
 			if (verbose) feedback.printf("File: %s [%s]\n", unit.sourcefile.getName(), result.isChanged() ? "delomboked" : "unchanged");
 			Writer rawWriter;
-			if (presetWriter != null) rawWriter = presetWriter;
+			if (presetWriter != null) rawWriter = createUnicodeEscapeWriter(presetWriter);
 			else if (output == null) rawWriter = createStandardOutWriter();
 			else rawWriter = createFileWriter(output, baseMap.get(unit), unit.sourcefile.toUri());
 			BufferedWriter writer = new BufferedWriter(rawWriter);
@@ -605,6 +605,10 @@ public class Delombok {
 		return createUnicodeEscapeWriter(System.out);
 	}
 	
+	private Writer createUnicodeEscapeWriter(Writer writer) {
+		return new UnicodeEscapeWriter(writer, charset);
+	}
+	
 	private Writer createUnicodeEscapeWriter(OutputStream out) {
 		return new UnicodeEscapeWriter(new OutputStreamWriter(out, charset), charset);
 	}

src/delombok/lombok/delombok/PrettyCommentsPrinter.java

@@ -104,7 +104,6 @@ import com.sun.tools.javac.tree.JCTree.LetExpr;
 import com.sun.tools.javac.tree.JCTree.TypeBoundKind;
 import com.sun.tools.javac.tree.TreeInfo;
 import com.sun.tools.javac.tree.TreeScanner;
-import com.sun.tools.javac.util.Convert;
 import com.sun.tools.javac.util.List;
 import com.sun.tools.javac.util.Name;
 import com.sun.tools.javac.util.Position;
@@ -209,7 +208,6 @@ public class PrettyCommentsPrinter extends JCTree.Visitor {
 	
 	private void consumeComments(int until, JCTree tree) throws IOException {
 		boolean prevNewLine = onNewLine;
-		boolean found = false;
 		CommentInfo head = comments.head;
 		while (comments.nonEmpty() && head.pos < until) {
 			printComment(head);
@@ -355,7 +353,7 @@ public class PrettyCommentsPrinter extends JCTree.Visitor {
 		}
 		needsSpace = false;
 		
-		out.write(Convert.escapeUnicode(s.toString()));
+		out.write(s.toString());
 		
 		onNewLine = false;
 		aligned = false;
@@ -1413,16 +1411,51 @@ public class PrettyCommentsPrinter extends JCTree.Visitor {
 			else if (CTC_FLOAT.equals(typeTag)) print(tree.value + "F");
 			else if (CTC_DOUBLE.equals(typeTag)) print(tree.value.toString());
 			else if (CTC_CHAR.equals(typeTag)) {
-				print("\'" + Convert.quote(String.valueOf((char)((Number)tree.value).intValue())) + "\'");
+				print("\'" + quoteChar((char)((Number)tree.value).intValue()) + "\'");
 			}
 			else if (CTC_BOOLEAN.equals(typeTag)) print(((Number)tree.value).intValue() == 1 ? "true" : "false");
 			else if (CTC_BOT.equals(typeTag)) print("null");
-			else print("\"" + Convert.quote(tree.value.toString()) + "\"");
+			else print("\"" + quoteChars(tree.value.toString()) + "\"");
 		} catch (IOException e) {
 			throw new UncheckedIOException(e);
 		}
 	}
 	
+	public static String quoteChars(String s) {
+		StringBuilder buf = new StringBuilder();
+		for (int i = 0; i < s.length(); i++) {
+			buf.append(quoteChar(s.charAt(i)));
+		}
+		return buf.toString();
+	}
+	
+	/**
+	 * Escapes a character if it has an escape sequence or is non-printable
+	 * ASCII. Leaves non-ASCII characters alone.
+	 */
+	public static String quoteChar(char ch) {
+		switch (ch) {
+		case '\b':
+			return "\\b";
+		case '\f':
+			return "\\f";
+		case '\n':
+			return "\\n";
+		case '\r':
+			return "\\r";
+		case '\t':
+			return "\\t";
+		case '\'':
+			return "\\'";
+		case '\"':
+			return "\\\"";
+		case '\\':
+			return "\\\\";
+		default:
+			return ch < 32 ? String.format("\\%03o", (int) ch) : String.valueOf(ch);
+		}
+	}
+	
 	public void visitTypeIdent(JCPrimitiveTypeTree tree) {
 		TypeTag typetag = typeTag(tree);
 		try {

src/delombok/lombok/delombok/UnicodeEscapeWriter.java

@@ -63,6 +63,6 @@ public class UnicodeEscapeWriter extends Writer {
 	}
 	
 	protected void writeUnicodeEscape(char c) throws IOException {
-		writer.write("\\u" + Integer.toHexString(c));
+		writer.write(String.format("\\u%04x", (int) c));
 	}
 }

test/core/src/lombok/AbstractRunTests.java

@@ -75,7 +75,7 @@ public abstract class AbstractRunTests {
 			}
 		});
 		
-		transformCode(messages, writer, file);
+		transformCode(messages, writer, file, sourceDirectives.getSpecifiedEncoding());
 		
 		compare(file.getName(), expected, writer.toString(), messages, params.printErrors(), sourceDirectives.isSkipCompareContent() || expected.isSkipCompareContent());
 		return true;
@@ -97,7 +97,7 @@ public abstract class AbstractRunTests {
 		return 8;
 	}
 	
-	protected abstract void transformCode(Collection<CompilerMessage> messages, StringWriter result, File file) throws Throwable;
+	protected abstract void transformCode(Collection<CompilerMessage> messages, StringWriter result, File file, String encoding) throws Throwable;
 	
 	protected String readFile(File file) throws IOException {
 		BufferedReader reader;

test/core/src/lombok/LombokTestSource.java

@@ -49,7 +49,8 @@ public class LombokTestSource {
 	private final boolean skipCompareContent;
 	private final int versionLowerLimit, versionUpperLimit;
 	private final ConfigurationResolver configuration;
-	
+	private final String specifiedEncoding;
+
 	public boolean versionWithinLimit(int version) {
 		return version >= versionLowerLimit && version <= versionUpperLimit;
 	}
@@ -74,6 +75,10 @@ public class LombokTestSource {
 		return skipCompareContent;
 	}
 	
+	public String getSpecifiedEncoding() {
+		return specifiedEncoding;
+	}
+	
 	public ConfigurationResolver getConfiguration() {
 		return configuration;
 	}
@@ -123,6 +128,7 @@ public class LombokTestSource {
 		int versionUpper = Integer.MAX_VALUE;
 		boolean ignore = false;
 		boolean skipCompareContent = false;
+		String encoding = null;
 		
 		for (String directive : directives) {
 			directive = directive.trim();
@@ -154,10 +160,15 @@ public class LombokTestSource {
 				continue;
 			}
 			
+			if (lc.startsWith("encoding:")) {
+				encoding = directive.substring(9).trim();
+				continue;
+			}
+			
 			Assert.fail("Directive line \"" + directive + "\" in '" + file.getAbsolutePath() + "' invalid: unrecognized directive.");
 			throw new RuntimeException();
 		}
-		
+		this.specifiedEncoding = encoding;
 		this.versionLowerLimit = versionLower;
 		this.versionUpperLimit = versionUpper;
 		this.ignore = ignore;
@@ -194,13 +205,17 @@ public class LombokTestSource {
 	}
 	
 	public static LombokTestSource read(File sourceFolder, File messagesFolder, String fileName) throws IOException {
+		return read0(sourceFolder, messagesFolder, fileName, "UTF-8");
+	}
+	
+	private static LombokTestSource read0(File sourceFolder, File messagesFolder, String fileName, String encoding) throws IOException {
 		StringBuilder content = null;
 		List<String> directives = new ArrayList<String>();
 		
 		File sourceFile = new File(sourceFolder, fileName);
 		if (sourceFile.exists()) {
 			@Cleanup val rawIn = new FileInputStream(sourceFile);
-			BufferedReader in = new BufferedReader(new InputStreamReader(rawIn, "UTF-8"));
+			BufferedReader in = new BufferedReader(new InputStreamReader(rawIn, encoding));
 			for (String i = in.readLine(); i != null; i = in.readLine()) {
 				if (content != null) {
 					content.append(i).append("\n");
@@ -234,6 +249,15 @@ public class LombokTestSource {
 			}
 		}
 		
-		return new LombokTestSource(sourceFile, content.toString(), messages, directives);
+		LombokTestSource source = new LombokTestSource(sourceFile, content.toString(), messages, directives);
+		String specifiedEncoding = source.getSpecifiedEncoding();
+		
+		// The source file has an 'encoding' header to test encoding issues. Of course, reading the encoding header
+		// requires knowing the encoding of the file first. In practice we get away with it, because UTF-8 and US-ASCII are compatible enough.
+		// The fix is therefore to read in as UTF-8 initially, and if the file requests that it should be read as another encoding, toss it all
+		// and reread that way.
+		
+		if (specifiedEncoding == null || specifiedEncoding.equalsIgnoreCase(encoding)) return source;
+		return read0(sourceFolder, messagesFolder, fileName, specifiedEncoding);
 	}
 }

test/core/src/lombok/RunTestsViaDelombok.java

@@ -34,10 +34,10 @@ public class RunTestsViaDelombok extends AbstractRunTests {
 	private Delombok delombok = new Delombok();
 	
 	@Override
-	public void transformCode(Collection<CompilerMessage> messages, StringWriter result, final File file) throws Throwable {
+	public void transformCode(Collection<CompilerMessage> messages, StringWriter result, final File file, String encoding) throws Throwable {
 		delombok.setVerbose(false);
 		delombok.setForceProcess(true);
-		delombok.setCharset("UTF-8");
+		delombok.setCharset(encoding == null ? "UTF-8" : encoding);
 		
 		delombok.setDiagnosticsListener(new CapturingDiagnosticListener(file, messages));
 		

test/core/src/lombok/RunTestsViaEcj.java

@@ -93,7 +93,7 @@ public class RunTestsViaEcj extends AbstractRunTests {
 	}
 	
 	@Override
-	public void transformCode(Collection<CompilerMessage> messages, StringWriter result, File file) throws Throwable {
+	public void transformCode(Collection<CompilerMessage> messages, StringWriter result, File file, String encoding) throws Throwable {
 		final AtomicReference<CompilationResult> compilationResult_ = new AtomicReference<CompilationResult>();
 		final AtomicReference<CompilationUnitDeclaration> compilationUnit_ = new AtomicReference<CompilationUnitDeclaration>();
 		ICompilerRequestor bitbucketRequestor = new ICompilerRequestor() {
@@ -103,7 +103,7 @@ public class RunTestsViaEcj extends AbstractRunTests {
 		};
 		
 		String source = readFile(file);
-		final CompilationUnit sourceUnit = new CompilationUnit(source.toCharArray(), file.getName(), "UTF-8");
+		final CompilationUnit sourceUnit = new CompilationUnit(source.toCharArray(), file.getName(), encoding == null ? "UTF-8" : encoding);
 		
 		Compiler ecjCompiler = new Compiler(createFileSystem(file), ecjErrorHandlingPolicy(), ecjCompilerOptions(), bitbucketRequestor, new DefaultProblemFactory(Locale.ENGLISH)) {
 			@Override protected synchronized void addCompilationUnit(ICompilationUnit inUnit, CompilationUnitDeclaration parsedUnit) {

test/transform/resource/after-delombok/EncodingUsAscii.java

@@ -0,0 +1,9 @@
+//ENCODING: US-ASCII
+class EncodingUsAscii {
+	String foo\u0e51\u0e51 = "\016\t\b ";
+	@java.lang.Override
+	@java.lang.SuppressWarnings("all")
+	public java.lang.String toString() {
+		return "EncodingUsAscii(foo\u0e51\u0e51=" + this.foo\u0e51\u0e51 + ")";
+	}
+}

test/transform/resource/after-delombok/EncodingUtf8.java

@@ -0,0 +1,8 @@
+class EncodingUtf8 {
+	String foo๑๑ = "\016\t\b ";
+	@java.lang.Override
+	@java.lang.SuppressWarnings("all")
+	public java.lang.String toString() {
+		return "EncodingUtf8(foo๑๑=" + this.foo๑๑ + ")";
+	}
+}

test/transform/resource/after-ecj/EncodingUsAscii.java

@@ -0,0 +1 @@
+//ignore: This test serves to check what happens with 'weird' characters when you use delombok. It's just not relevant for ecj.

test/transform/resource/after-ecj/EncodingUtf8.java

@@ -0,0 +1,9 @@
+@lombok.ToString class EncodingUtf8 {
+  String foo๑๑ = "\t\b ";
+  EncodingUtf8() {
+    super();
+  }
+  public @java.lang.Override @java.lang.SuppressWarnings("all") java.lang.String toString() {
+    return (("EncodingUtf8(foo๑๑=" + this.foo๑๑) + ")");
+  }
+}

test/transform/resource/before/EncodingUsAscii.java

@@ -0,0 +1,5 @@
+//ENCODING: US-ASCII
+@lombok.ToString
+class EncodingUsAscii {
+	String foo\u0e51\u0e51 = "\u000e	\10 ";
+}

test/transform/resource/before/EncodingUtf8.java

@@ -0,0 +1,5 @@
+//ENCODING: UTF-8
+@lombok.ToString
+class EncodingUtf8 {
+	String foo\u0e51๑ = "\u000e	\10 ";
+}