fhueske commented on a change in pull request #6823: [FLINK-10134] UTF-16 support for TextInputFormat bug refixed URL: https://github.com/apache/flink/pull/6823#discussion_r225574255
########## File path: flink-java/src/test/java/org/apache/flink/api/java/io/TextInputFormatTest.java ########## @@ -207,12 +207,212 @@ private void testRemovingTrailingCR(String lineBreaker, String delimiter) { assertEquals(content, result); } + } catch (Throwable t) { + System.err.println("test failed with exception: " + t.getMessage()); + t.printStackTrace(System.err); + fail("Test erroneous"); } - catch (Throwable t) { + } + + /** + * Test different file encodings,for example: UTF-8, UTF-8 with bom, UTF-16LE, UTF-16BE, UTF-32LE, UTF-32BE. + */ + @Test + public void testFileCharset() { + String first = "First line"; + + // Test special different languages + for (final String data : new String[]{"Hello", "ハロー", "привет", "Bonjour", "Сайн байна уу", "안녕하세요."}) { + testAllFileCharsetNoDelimiter(data); + } + + // Test special symbol + for (final String delimiterStr : new String[]{"\\", "^", "|", "[", ".", "*"}) { + first = "Fir" + delimiterStr + "st li" + delimiterStr + "ne"; + testAllFileCharsetWithDelimiter(first, delimiterStr); + } + } + + private void testAllFileCharsetNoDelimiter(String first) { + testAllFileCharsetWithDelimiter(first, ""); + } + + private void testAllFileCharsetWithDelimiter(String first, String delimiter) { + try { + final byte[] noBom = new byte[]{}; Review comment: Move the bom variables to `testFileCharset` and change its `bom` parameter to `boolean hasBom` ---------------------------------------------------------------- This is an automated message from the Apache Git Service. To respond to the message, please log on GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org With regards, Apache Git Services