SLIDER-1032 ApplicationWithAddonPackagesIT failing (against windows)
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/d497731e Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/d497731e Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/d497731e Branch: refs/heads/develop Commit: d497731e4fa33640838d75bbe922e2974f240dad Parents: 520f644 Author: Steve Loughran <ste...@apache.org> Authored: Thu Dec 17 19:17:21 2015 +0000 Committer: Steve Loughran <ste...@apache.org> Committed: Thu Dec 17 19:17:21 2015 +0000 ---------------------------------------------------------------------- .../apache/slider/test/ContractTestUtils.java | 901 +++++++++++++++++++ .../framework/AgentCommandTestBase.groovy | 70 +- ...nentConfigsInAppConfigShowUpOnAgentIT.groovy | 2 +- .../ApplicationWithAddonPackagesIT.groovy | 67 +- 4 files changed, 990 insertions(+), 50 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/d497731e/slider-core/src/test/java/org/apache/slider/test/ContractTestUtils.java ---------------------------------------------------------------------- diff --git a/slider-core/src/test/java/org/apache/slider/test/ContractTestUtils.java b/slider-core/src/test/java/org/apache/slider/test/ContractTestUtils.java new file mode 100644 index 0000000..7eaaefe --- /dev/null +++ b/slider-core/src/test/java/org/apache/slider/test/ContractTestUtils.java @@ -0,0 +1,901 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.slider.test; + +import org.apache.hadoop.fs.FSDataInputStream; +import org.apache.hadoop.fs.FSDataOutputStream; +import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.junit.Assert; +import org.junit.internal.AssumptionViolatedException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.EOFException; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.util.Arrays; +import java.util.Properties; +import java.util.UUID; + +/** + * Utilities used across test cases to make assertions about filesystems + * -assertions which fail with useful information. + * This is lifted from Hadoop common Test; that JAR isn't published, so + * we have to make do. + */ +public class ContractTestUtils extends Assert { + + private static final Logger LOG = + LoggerFactory.getLogger(ContractTestUtils.class); + + public static final String IO_FILE_BUFFER_SIZE = "io.file.buffer.size"; + + // For scale testing, we can repeatedly write small chunk data to generate + // a large file. + public static final String IO_CHUNK_BUFFER_SIZE = "io.chunk.buffer.size"; + public static final int DEFAULT_IO_CHUNK_BUFFER_SIZE = 128; + public static final String IO_CHUNK_MODULUS_SIZE = "io.chunk.modulus.size"; + public static final int DEFAULT_IO_CHUNK_MODULUS_SIZE = 128; + + /** + * Assert that a property in the property set matches the expected value + * @param props property set + * @param key property name + * @param expected expected value. If null, the property must not be in the set + */ + public static void assertPropertyEquals(Properties props, + String key, + String expected) { + String val = props.getProperty(key); + if (expected == null) { + assertNull("Non null property " + key + " = " + val, val); + } else { + assertEquals("property " + key + " = " + val, + expected, + val); + } + } + + /** + * + * Write a file and read it in, validating the result. Optional flags control + * whether file overwrite operations should be enabled, and whether the + * file should be deleted afterwards. + * + * If there is a mismatch between what was written and what was expected, + * a small range of bytes either side of the first error are logged to aid + * diagnosing what problem occurred -whether it was a previous file + * or a corrupting of the current file. This assumes that two + * sequential runs to the same path use datasets with different character + * moduli. + * + * @param fs filesystem + * @param path path to write to + * @param len length of data + * @param overwrite should the create option allow overwrites? + * @param delete should the file be deleted afterwards? -with a verification + * that it worked. Deletion is not attempted if an assertion has failed + * earlier -it is not in a <code>finally{}</code> block. + * @throws IOException IO problems + */ + public static void writeAndRead(FileSystem fs, + Path path, + byte[] src, + int len, + int blocksize, + boolean overwrite, + boolean delete) throws IOException { + fs.mkdirs(path.getParent()); + + writeDataset(fs, path, src, len, blocksize, overwrite); + + byte[] dest = readDataset(fs, path, len); + + compareByteArrays(src, dest, len); + + if (delete) { + rejectRootOperation(path); + boolean deleted = fs.delete(path, false); + assertTrue("Deleted", deleted); + assertPathDoesNotExist(fs, "Cleanup failed", path); + } + } + + /** + * Write a file. + * Optional flags control + * whether file overwrite operations should be enabled + * @param fs filesystem + * @param path path to write to + * @param len length of data + * @param overwrite should the create option allow overwrites? + * @throws IOException IO problems + */ + public static void writeDataset(FileSystem fs, + Path path, + byte[] src, + int len, + int buffersize, + boolean overwrite) throws IOException { + assertTrue( + "Not enough data in source array to write " + len + " bytes", + src.length >= len); + FSDataOutputStream out = fs.create(path, + overwrite, + fs.getConf() + .getInt(IO_FILE_BUFFER_SIZE, + 4096), + (short) 1, + buffersize); + out.write(src, 0, len); + out.close(); + assertFileHasLength(fs, path, len); + } + + /** + * Read the file and convert to a byte dataset. + * This implements readfully internally, so that it will read + * in the file without ever having to seek() + * @param fs filesystem + * @param path path to read from + * @param len length of data to read + * @return the bytes + * @throws IOException IO problems + */ + public static byte[] readDataset(FileSystem fs, Path path, int len) + throws IOException { + FSDataInputStream in = fs.open(path); + byte[] dest = new byte[len]; + int offset =0; + int nread = 0; + try { + while (nread < len) { + int nbytes = in.read(dest, offset + nread, len - nread); + if (nbytes < 0) { + throw new EOFException("End of file reached before reading fully."); + } + nread += nbytes; + } + } finally { + in.close(); + } + return dest; + } + + /** + * Read a file, verify its length and contents match the expected array + * @param fs filesystem + * @param path path to file + * @param original original dataset + * @throws IOException IO Problems + */ + public static void verifyFileContents(FileSystem fs, + Path path, + byte[] original) throws IOException { + FileStatus stat = fs.getFileStatus(path); + String statText = stat.toString(); + assertTrue("not a file " + statText, stat.isFile()); + assertEquals("wrong length " + statText, original.length, stat.getLen()); + byte[] bytes = readDataset(fs, path, original.length); + compareByteArrays(original,bytes,original.length); + } + + /** + * Verify that the read at a specific offset in a stream + * matches that expected + * @param stm stream + * @param fileContents original file contents + * @param seekOff seek offset + * @param toRead number of bytes to read + * @throws IOException IO problems + */ + public static void verifyRead(FSDataInputStream stm, byte[] fileContents, + int seekOff, int toRead) throws IOException { + byte[] out = new byte[toRead]; + stm.seek(seekOff); + stm.readFully(out); + byte[] expected = Arrays.copyOfRange(fileContents, seekOff, + seekOff + toRead); + compareByteArrays(expected, out,toRead); + } + + /** + * Assert that tthe array original[0..len] and received[] are equal. + * A failure triggers the logging of the bytes near where the first + * difference surfaces. + * @param original source data + * @param received actual + * @param len length of bytes to compare + */ + public static void compareByteArrays(byte[] original, + byte[] received, + int len) { + assertEquals("Number of bytes read != number written", + len, received.length); + int errors = 0; + int first_error_byte = -1; + for (int i = 0; i < len; i++) { + if (original[i] != received[i]) { + if (errors == 0) { + first_error_byte = i; + } + errors++; + } + } + + if (errors > 0) { + String message = String.format(" %d errors in file of length %d", + errors, len); + LOG.warn(message); + // the range either side of the first error to print + // this is a purely arbitrary number, to aid user debugging + final int overlap = 10; + for (int i = Math.max(0, first_error_byte - overlap); + i < Math.min(first_error_byte + overlap, len); + i++) { + byte actual = received[i]; + byte expected = original[i]; + String letter = toChar(actual); + String line = String.format("[%04d] %2x %s\n", i, actual, letter); + if (expected != actual) { + line = String.format("[%04d] %2x %s -expected %2x %s\n", + i, + actual, + letter, + expected, + toChar(expected)); + } + LOG.warn(line); + } + fail(message); + } + } + + /** + * Convert a byte to a character for printing. If the + * byte value is < 32 -and hence unprintable- the byte is + * returned as a two digit hex value + * @param b byte + * @return the printable character string + */ + public static String toChar(byte b) { + if (b >= 0x20) { + return Character.toString((char) b); + } else { + return String.format("%02x", b); + } + } + + /** + * Convert a buffer to a string, character by character + * @param buffer input bytes + * @return a string conversion + */ + public static String toChar(byte[] buffer) { + StringBuilder builder = new StringBuilder(buffer.length); + for (byte b : buffer) { + builder.append(toChar(b)); + } + return builder.toString(); + } + + public static byte[] toAsciiByteArray(String s) { + char[] chars = s.toCharArray(); + int len = chars.length; + byte[] buffer = new byte[len]; + for (int i = 0; i < len; i++) { + buffer[i] = (byte) (chars[i] & 0xff); + } + return buffer; + } + + /** + * Cleanup at the end of a test run + * @param action action triggering the operation (for use in logging) + * @param fileSystem filesystem to work with. May be null + * @param cleanupPath path to delete as a string + */ + public static void cleanup(String action, + FileSystem fileSystem, + String cleanupPath) { + if (fileSystem == null) { + return; + } + Path path = new Path(cleanupPath).makeQualified(fileSystem.getUri(), + fileSystem.getWorkingDirectory()); + cleanup(action, fileSystem, path); + } + + /** + * Cleanup at the end of a test run + * @param action action triggering the operation (for use in logging) + * @param fileSystem filesystem to work with. May be null + * @param path path to delete + */ + public static void cleanup(String action, FileSystem fileSystem, Path path) { + noteAction(action); + try { + rm(fileSystem, path, true, false); + } catch (Exception e) { + LOG.error("Error deleting in "+ action + " - " + path + ": " + e, e); + } + } + + /** + * Delete a directory. There's a safety check for operations against the + * root directory -these are intercepted and rejected with an IOException + * unless the allowRootDelete flag is true + * @param fileSystem filesystem to work with. May be null + * @param path path to delete + * @param recursive flag to enable recursive delete + * @param allowRootDelete can the root directory be deleted? + * @throws IOException on any problem. + */ + public static boolean rm(FileSystem fileSystem, + Path path, + boolean recursive, + boolean allowRootDelete) throws + IOException { + if (fileSystem != null) { + rejectRootOperation(path, allowRootDelete); + if (fileSystem.exists(path)) { + return fileSystem.delete(path, recursive); + } + } + return false; + + } + + /** + * Block any operation on the root path. This is a safety check + * @param path path in the filesystem + * @param allowRootOperation can the root directory be manipulated? + * @throws IOException if the operation was rejected + */ + public static void rejectRootOperation(Path path, + boolean allowRootOperation) throws IOException { + if (path.isRoot() && !allowRootOperation) { + throw new IOException("Root directory operation rejected: " + path); + } + } + + /** + * Block any operation on the root path. This is a safety check + * @param path path in the filesystem + * @throws IOException if the operation was rejected + */ + public static void rejectRootOperation(Path path) throws IOException { + rejectRootOperation(path, false); + } + + + public static void noteAction(String action) { + if (LOG.isDebugEnabled()) { + LOG.debug("============== "+ action +" ============="); + } + } + + /** + * downgrade a failure to a message and a warning, then an + * exception for the Junit test runner to mark as failed + * @param message text message + * @param failure what failed + * @throws AssumptionViolatedException always + */ + public static void downgrade(String message, Throwable failure) { + LOG.warn("Downgrading test " + message, failure); + AssumptionViolatedException ave = + new AssumptionViolatedException(failure, null); + throw ave; + } + + /** + * report an overridden test as unsupported + * @param message message to use in the text + * @throws AssumptionViolatedException always + */ + public static void unsupported(String message) { + skip(message); + } + + /** + * report a test has been skipped for some reason + * @param message message to use in the text + * @throws AssumptionViolatedException always + */ + public static void skip(String message) { + LOG.info("Skipping: {}", message); + throw new AssumptionViolatedException(message); + } + + /** + * Fail with an exception that was received + * @param text text to use in the exception + * @param thrown a (possibly null) throwable to init the cause with + * @throws AssertionError with the text and throwable -always + */ + public static void fail(String text, Throwable thrown) { + AssertionError e = new AssertionError(text); + e.initCause(thrown); + throw e; + } + + /** + * Make an assertion about the length of a file + * @param fs filesystem + * @param path path of the file + * @param expected expected length + * @throws IOException on File IO problems + */ + public static void assertFileHasLength(FileSystem fs, Path path, + int expected) throws IOException { + FileStatus status = fs.getFileStatus(path); + assertEquals( + "Wrong file length of file " + path + " status: " + status, + expected, + status.getLen()); + } + + /** + * Assert that a path refers to a directory + * @param fs filesystem + * @param path path of the directory + * @throws IOException on File IO problems + */ + public static void assertIsDirectory(FileSystem fs, + Path path) throws IOException { + FileStatus fileStatus = fs.getFileStatus(path); + assertIsDirectory(fileStatus); + } + + /** + * Assert that a path refers to a directory + * @param fileStatus stats to check + */ + public static void assertIsDirectory(FileStatus fileStatus) { + assertTrue("Should be a directory -but isn't: " + fileStatus, + fileStatus.isDirectory()); + } + + /** + * Write the text to a file, returning the converted byte array + * for use in validating the round trip + * @param fs filesystem + * @param path path of file + * @param text text to write + * @param overwrite should the operation overwrite any existing file? + * @return the read bytes + * @throws IOException on IO problems + */ + public static byte[] writeTextFile(FileSystem fs, + Path path, + String text, + boolean overwrite) throws IOException { + byte[] bytes = new byte[0]; + if (text != null) { + bytes = toAsciiByteArray(text); + } + createFile(fs, path, overwrite, bytes); + return bytes; + } + + /** + * Create a file + * @param fs filesystem + * @param path path to write + * @param overwrite overwrite flag + * @param data source dataset. Can be null + * @throws IOException on any problem + */ + public static void createFile(FileSystem fs, + Path path, + boolean overwrite, + byte[] data) throws IOException { + FSDataOutputStream stream = fs.create(path, overwrite); + if (data != null && data.length > 0) { + stream.write(data); + } + stream.close(); + } + + /** + * Touch a file + * @param fs filesystem + * @param path path + * @throws IOException IO problems + */ + public static void touch(FileSystem fs, + Path path) throws IOException { + createFile(fs, path, true, null); + } + + /** + * Delete a file/dir and assert that delete() returned true + * <i>and</i> that the path no longer exists. This variant rejects + * all operations on root directories + * @param fs filesystem + * @param file path to delete + * @param recursive flag to enable recursive delete + * @throws IOException IO problems + */ + public static void assertDeleted(FileSystem fs, + Path file, + boolean recursive) throws IOException { + assertDeleted(fs, file, recursive, false); + } + + /** + * Delete a file/dir and assert that delete() returned true + * <i>and</i> that the path no longer exists. This variant rejects + * all operations on root directories + * @param fs filesystem + * @param file path to delete + * @param recursive flag to enable recursive delete + * @param allowRootOperations can the root dir be deleted? + * @throws IOException IO problems + */ + public static void assertDeleted(FileSystem fs, + Path file, + boolean recursive, + boolean allowRootOperations) throws IOException { + rejectRootOperation(file, allowRootOperations); + assertPathExists(fs, "about to be deleted file", file); + boolean deleted = fs.delete(file, recursive); + String dir = ls(fs, file.getParent()); + assertTrue("Delete failed on " + file + ": " + dir, deleted); + assertPathDoesNotExist(fs, "Deleted file", file); + } + + /** + * Read in "length" bytes, convert to an ascii string + * @param fs filesystem + * @param path path to read + * @param length #of bytes to read. + * @return the bytes read and converted to a string + * @throws IOException IO problems + */ + public static String readBytesToString(FileSystem fs, + Path path, + int length) throws IOException { + FSDataInputStream in = fs.open(path); + try { + byte[] buf = new byte[length]; + in.readFully(0, buf); + return toChar(buf); + } finally { + in.close(); + } + } + + /** + * Take an array of filestats and convert to a string (prefixed w/ a [01] counter + * @param stats array of stats + * @param separator separator after every entry + * @return a stringified set + */ + public static String fileStatsToString(FileStatus[] stats, String separator) { + StringBuilder buf = new StringBuilder(stats.length * 128); + for (int i = 0; i < stats.length; i++) { + buf.append(String.format("[%02d] %s", i, stats[i])).append(separator); + } + return buf.toString(); + } + + /** + * List a directory + * @param fileSystem FS + * @param path path + * @return a directory listing or failure message + * @throws IOException + */ + public static String ls(FileSystem fileSystem, Path path) throws IOException { + if (path == null) { + //surfaces when someone calls getParent() on something at the top of the path + return "/"; + } + FileStatus[] stats; + String pathtext = "ls " + path; + try { + stats = fileSystem.listStatus(path); + } catch (FileNotFoundException e) { + return pathtext + " -file not found"; + } catch (IOException e) { + return pathtext + " -failed: " + e; + } + return dumpStats(pathtext, stats); + } + + public static String dumpStats(String pathname, FileStatus[] stats) { + return pathname + fileStatsToString(stats, "\n"); + } + + /** + * Assert that a file exists and whose {@link FileStatus} entry + * declares that this is a file and not a symlink or directory. + * @param fileSystem filesystem to resolve path against + * @param filename name of the file + * @throws IOException IO problems during file operations + */ + public static void assertIsFile(FileSystem fileSystem, Path filename) throws + IOException { + assertPathExists(fileSystem, "Expected file", filename); + FileStatus status = fileSystem.getFileStatus(filename); + assertIsFile(filename, status); + } + + /** + * Assert that a file exists and whose {@link FileStatus} entry + * declares that this is a file and not a symlink or directory. + * @param filename name of the file + * @param status file status + */ + public static void assertIsFile(Path filename, FileStatus status) { + String fileInfo = filename + " " + status; + assertFalse("File claims to be a directory " + fileInfo, + status.isDirectory()); + assertFalse("File claims to be a symlink " + fileInfo, + status.isSymlink()); + } + + /** + * Create a dataset for use in the tests; all data is in the range + * base to (base+modulo-1) inclusive + * @param len length of data + * @param base base of the data + * @param modulo the modulo + * @return the newly generated dataset + */ + public static byte[] dataset(int len, int base, int modulo) { + byte[] dataset = new byte[len]; + for (int i = 0; i < len; i++) { + dataset[i] = (byte) (base + (i % modulo)); + } + return dataset; + } + + /** + * Assert that a path exists -but make no assertions as to the + * type of that entry + * + * @param fileSystem filesystem to examine + * @param message message to include in the assertion failure message + * @param path path in the filesystem + * @throws FileNotFoundException raised if the path is missing + * @throws IOException IO problems + */ + public static void assertPathExists(FileSystem fileSystem, String message, + Path path) throws IOException { + if (!fileSystem.exists(path)) { + //failure, report it + String listing = ls(fileSystem, path.getParent()); + throw new FileNotFoundException(message + ": not found " + path + + " in \"" + path.getParent() + "\" :\n" + listing); + } + } + + /** + * Assert that a path does not exist + * + * @param fileSystem filesystem to examine + * @param message message to include in the assertion failure message + * @param path path in the filesystem + * @throws IOException IO problems + */ + public static void assertPathDoesNotExist(FileSystem fileSystem, + String message, + Path path) throws IOException { + try { + FileStatus status = fileSystem.getFileStatus(path); + fail(message + ": unexpectedly found " + path + " as " + status); + } catch (FileNotFoundException expected) { + //this is expected + + } + } + + /** + * Assert that a FileSystem.listStatus on a dir finds the subdir/child entry + * @param fs filesystem + * @param dir directory to scan + * @param subdir full path to look for + * @throws IOException IO probles + */ + public static void assertListStatusFinds(FileSystem fs, + Path dir, + Path subdir) throws IOException { + FileStatus[] stats = fs.listStatus(dir); + boolean found = false; + StringBuilder builder = new StringBuilder(); + for (FileStatus stat : stats) { + builder.append(stat.toString()).append('\n'); + if (stat.getPath().equals(subdir)) { + found = true; + } + } + assertTrue("Path " + subdir + + " not found in directory " + dir + ":" + builder, + found); + } + + /** + * Test for the host being an OSX machine + * @return true if the JVM thinks that is running on OSX + */ + public static boolean isOSX() { + return System.getProperty("os.name").contains("OS X"); + } + + /** + * compare content of file operations using a double byte array + * @param concat concatenated files + * @param bytes bytes + */ + public static void validateFileContent(byte[] concat, byte[][] bytes) { + int idx = 0; + boolean mismatch = false; + + for (byte[] bb : bytes) { + for (byte b : bb) { + if (b != concat[idx++]) { + mismatch = true; + break; + } + } + if (mismatch) + break; + } + assertFalse("File content of file is not as expected at offset " + idx, + mismatch); + } + + /** + * Receives test data from the given input file and checks the size of the + * data as well as the pattern inside the received data. + * + * @param fs FileSystem + * @param path Input file to be checked + * @param expectedSize the expected size of the data to be read from the + * input file in bytes + * @param bufferLen Pattern length + * @param modulus Pattern modulus + * @throws IOException + * thrown if an error occurs while reading the data + */ + public static void verifyReceivedData(FileSystem fs, Path path, + final long expectedSize, + final int bufferLen, + final int modulus) throws IOException { + final byte[] testBuffer = new byte[bufferLen]; + + long totalBytesRead = 0; + int nextExpectedNumber = 0; + final InputStream inputStream = fs.open(path); + try { + while (true) { + final int bytesRead = inputStream.read(testBuffer); + if (bytesRead < 0) { + break; + } + + totalBytesRead += bytesRead; + + for (int i = 0; i < bytesRead; ++i) { + if (testBuffer[i] != nextExpectedNumber) { + throw new IOException("Read number " + testBuffer[i] + + " but expected " + nextExpectedNumber); + } + + ++nextExpectedNumber; + + if (nextExpectedNumber == modulus) { + nextExpectedNumber = 0; + } + } + } + + if (totalBytesRead != expectedSize) { + throw new IOException("Expected to read " + expectedSize + + " bytes but only received " + totalBytesRead); + } + } finally { + inputStream.close(); + } + } + + /** + * Generates test data of the given size according to some specific pattern + * and writes it to the provided output file. + * + * @param fs FileSystem + * @param path Test file to be generated + * @param size The size of the test data to be generated in bytes + * @param bufferLen Pattern length + * @param modulus Pattern modulus + * @throws IOException + * thrown if an error occurs while writing the data + */ + public static long generateTestFile(FileSystem fs, Path path, + final long size, + final int bufferLen, + final int modulus) throws IOException { + final byte[] testBuffer = new byte[bufferLen]; + for (int i = 0; i < testBuffer.length; ++i) { + testBuffer[i] = (byte) (i % modulus); + } + + final OutputStream outputStream = fs.create(path, false); + long bytesWritten = 0; + try { + while (bytesWritten < size) { + final long diff = size - bytesWritten; + if (diff < testBuffer.length) { + outputStream.write(testBuffer, 0, (int) diff); + bytesWritten += diff; + } else { + outputStream.write(testBuffer); + bytesWritten += testBuffer.length; + } + } + + return bytesWritten; + } finally { + outputStream.close(); + } + } + + /** + * Creates and reads a file with the given size. The test file is generated + * according to a specific pattern so it can be easily verified even if it's + * a multi-GB one. + * During the read phase the incoming data stream is also checked against + * this pattern. + * + * @param fs FileSystem + * @param parent Test file parent dir path + * @throws IOException + * thrown if an I/O error occurs while writing or reading the test file + */ + public static void createAndVerifyFile(FileSystem fs, Path parent, final long fileSize) + throws IOException { + int testBufferSize = fs.getConf() + .getInt(IO_CHUNK_BUFFER_SIZE, DEFAULT_IO_CHUNK_BUFFER_SIZE); + int modulus = fs.getConf() + .getInt(IO_CHUNK_MODULUS_SIZE, DEFAULT_IO_CHUNK_MODULUS_SIZE); + + final String objectName = UUID.randomUUID().toString(); + final Path objectPath = new Path(parent, objectName); + + // Write test file in a specific pattern + assertEquals(fileSize, + generateTestFile(fs, objectPath, fileSize, testBufferSize, modulus)); + assertPathExists(fs, "not created successful", objectPath); + + // Now read the same file back and verify its content + try { + verifyReceivedData(fs, objectPath, fileSize, testBufferSize, modulus); + } finally { + // Delete test file + fs.delete(objectPath, false); + } + } +} http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/d497731e/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/AgentCommandTestBase.groovy ---------------------------------------------------------------------- diff --git a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/AgentCommandTestBase.groovy b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/AgentCommandTestBase.groovy index 12890b7..ec4b03a 100644 --- a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/AgentCommandTestBase.groovy +++ b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/AgentCommandTestBase.groovy @@ -19,13 +19,15 @@ package org.apache.slider.funtest.framework import groovy.util.logging.Slf4j +import org.apache.hadoop.fs.FileSystem import org.apache.hadoop.fs.Path import org.apache.hadoop.security.UserGroupInformation import org.apache.hadoop.yarn.api.records.YarnApplicationState -import org.apache.hadoop.fs.FileSystem; import org.apache.slider.common.SliderExitCodes import org.apache.slider.common.params.Arguments import org.apache.slider.common.params.SliderActions +import org.apache.slider.test.ContractTestUtils +import org.apache.slider.test.Outcome import org.apache.tools.zip.ZipEntry import org.apache.tools.zip.ZipOutputStream import org.junit.Before @@ -151,30 +153,52 @@ implements FuntestProperties, Arguments, SliderExitCodes, SliderActions { } } } - - protected static void verifyFileExist(String filePath){ - try{ - Path pt = new Path(filePath); - def uploader = new FileUploader(SLIDER_CONFIG, - UserGroupInformation.currentUser) - FileSystem fs = uploader.getFileSystem(); - assert fs.exists(pt); - }catch(IOException e){ - log.error("IOException during verifying file exist " + e.toString()); + + /** + * Probe callback for the the app root web page up. + * Raising an SSL exception is considered a sign of liveness. + * @param args map where 'applicationId' must be set + * @return the outcome + */ + protected static Outcome doesFileExist( + Map<String, String> args) { + return fileExists(args['filename']) ? Outcome.Success : Outcome.Retry + } + + /** + * Await for a file to exist + * @param filename file to look for + * @param timeout launch timeout + */ + void awaitFileExists(String filename, int timeout) { + repeatUntilSuccess( + "await File existence", + this.&doesFileExist, + timeout, + PROBE_SLEEP_TIME, + [ + filename: filename + ], + true, + "file $filename not found") { + verifyFileExists(filename, "After timeout of $timeout ms") } } - - protected static void cleanupHdfsFile(String filePath){ - try{ - Path pt = new Path(filePath); - def uploader = new FileUploader(SLIDER_CONFIG, - UserGroupInformation.currentUser) - FileSystem fs = uploader.getFileSystem(); - if( fs.exists(pt)){ - fs.delete(pt, false); - } - }catch(IOException e){ - log.error("IOException during deleting file: " + e.toString()); + + protected static boolean fileExists(String filePath) { + assert filePath + clusterFS.exists(new Path(filePath)); + } + + protected static void verifyFileExists(String filePath, String message = "") { + ContractTestUtils.assertPathExists(clusterFS, message, new Path(filePath)) + } + + protected static void cleanupHdfsFile(String filePath) { + Path pt = new Path(filePath); + FileSystem fs = clusterFS + if (fs.exists(pt)) { + fs.delete(pt, false); } } http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/d497731e/slider-funtest/src/test/groovy/org/apache/slider/funtest/basic/ComponentConfigsInAppConfigShowUpOnAgentIT.groovy ---------------------------------------------------------------------- diff --git a/slider-funtest/src/test/groovy/org/apache/slider/funtest/basic/ComponentConfigsInAppConfigShowUpOnAgentIT.groovy b/slider-funtest/src/test/groovy/org/apache/slider/funtest/basic/ComponentConfigsInAppConfigShowUpOnAgentIT.groovy index 91797f9..0d0eab2 100644 --- a/slider-funtest/src/test/groovy/org/apache/slider/funtest/basic/ComponentConfigsInAppConfigShowUpOnAgentIT.groovy +++ b/slider-funtest/src/test/groovy/org/apache/slider/funtest/basic/ComponentConfigsInAppConfigShowUpOnAgentIT.groovy @@ -78,7 +78,7 @@ public class ComponentConfigsInAppConfigShowUpOnAgentIT extends AgentCommandTest list(0, [ARG_STATE, "running"]) status(0, CLUSTER) Thread.sleep(10000) - verifyFileExist(TARGET_FILE) + verifyFileExists(TARGET_FILE) } public void setupApplicationPackage() { http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/d497731e/slider-funtest/src/test/groovy/org/apache/slider/funtest/coprocessors/ApplicationWithAddonPackagesIT.groovy ---------------------------------------------------------------------- diff --git a/slider-funtest/src/test/groovy/org/apache/slider/funtest/coprocessors/ApplicationWithAddonPackagesIT.groovy b/slider-funtest/src/test/groovy/org/apache/slider/funtest/coprocessors/ApplicationWithAddonPackagesIT.groovy index b32275e..66572ba 100644 --- a/slider-funtest/src/test/groovy/org/apache/slider/funtest/coprocessors/ApplicationWithAddonPackagesIT.groovy +++ b/slider-funtest/src/test/groovy/org/apache/slider/funtest/coprocessors/ApplicationWithAddonPackagesIT.groovy @@ -18,11 +18,14 @@ package org.apache.slider.funtest.coprocessors import groovy.transform.CompileStatic import groovy.util.logging.Slf4j +import org.apache.hadoop.fs.Path import org.apache.slider.common.params.Arguments import org.apache.slider.common.tools.SliderUtils import org.apache.slider.funtest.framework.SliderShell import org.apache.slider.funtest.framework.AgentCommandTestBase import org.apache.slider.funtest.framework.CommandTestBase +import org.apache.slider.test.ContractTestUtils +import org.apache.slider.test.Outcome import org.junit.After import org.junit.Before import org.junit.Test @@ -46,12 +49,15 @@ public class ApplicationWithAddonPackagesIT extends AgentCommandTestBase{ static String ADD_ON_PACKAGE_NO_COMPONENT_PKG_NAME = "add-on-package-apply-on-no-component" static String ADD_ON_PACKAGE_NO_COMPONENT_PKG_FILE = "target/package-tmp/add-on-package-apply-on-no-component.zip" static String TARGET_FILE = "/tmp/test_slider.txt" + public static final int FILE_EXISTS_TIMEOUT = 20000 protected String APP_RESOURCE = getAppResource() protected String APP_TEMPLATE = getAppTemplate() @Before public void prepareCluster() { setupCluster(CLUSTER) + cleanupHdfsFile(TARGET_FILE) + clusterFS.mkdirs(new Path(TARGET_FILE).getParent()) } @After @@ -86,30 +92,35 @@ public class ApplicationWithAddonPackagesIT extends AgentCommandTestBase{ list(0, [ARG_LIVE]) list(0, [ARG_STATE, "running"]) status(0, CLUSTER) - Thread.sleep(10000) - verifyFileExist(TARGET_FILE) + awaitTargetFileExists() } - + + protected void awaitTargetFileExists() { + awaitFileExists(TARGET_FILE, FILE_EXISTS_TIMEOUT) + } + @Test public void testCreateApplicationWithOneAddonPackagesForNoComponents() throws Throwable { describe("Create a cluster with an addon package that apply to no components") SliderUtils.zipFolder(new File(ADD_ON_PACKAGE_NO_COMPONENT), new File(ADD_ON_PACKAGE_NO_COMPONENT_PKG_FILE)) def clusterpath = buildClusterPath(CLUSTER) File launchReportFile = createTempJsonFile(); - cleanupHdfsFile(TARGET_FILE) - + //default waiting time too long, temporarily lower it int temp_holder = CommandTestBase.THAW_WAIT_TIME; CommandTestBase.THAW_WAIT_TIME = 100; - SliderShell shell = createTemplatedSliderApplication(CLUSTER, + SliderShell shell + try { + shell = createTemplatedSliderApplication(CLUSTER, APP_TEMPLATE, APP_RESOURCE2, [Arguments.ARG_ADDON, ADD_ON_PACKAGE_NO_COMPONENT_PKG_NAME, ADD_ON_PACKAGE_NO_COMPONENT_PKG_FILE], launchReportFile) - CommandTestBase.THAW_WAIT_TIME = temp_holder; + } finally { + CommandTestBase.THAW_WAIT_TIME = temp_holder; + } logShell(shell) - Thread.sleep(10000) //the Slider AM will fail while checking no components in metainfo.json of addon pkg // SLIDER-897 - Disabling this flaky assert. Have to re-write the test to @@ -124,8 +135,7 @@ public class ApplicationWithAddonPackagesIT extends AgentCommandTestBase{ SliderUtils.zipFolder(new File(ADD_ON_PACKAGE_MULTI_COMPONENT), new File(ADD_ON_PACKAGE_MULTI_COMPONENT_PKG_FILE)) def clusterpath = buildClusterPath(CLUSTER) File launchReportFile = createTempJsonFile(); - cleanupHdfsFile(TARGET_FILE) - + SliderShell shell = createTemplatedSliderApplication(CLUSTER, APP_TEMPLATE, APP_RESOURCE2, @@ -144,8 +154,8 @@ public class ApplicationWithAddonPackagesIT extends AgentCommandTestBase{ list(0, [ARG_LIVE]) list(0, [ARG_STATE, "running"]) status(0, CLUSTER) - Thread.sleep(10000) - verifyFileExist(TARGET_FILE) + awaitTargetFileExists() + } @Test @@ -154,8 +164,7 @@ public class ApplicationWithAddonPackagesIT extends AgentCommandTestBase{ SliderUtils.zipFolder(new File(ADD_ON_PACKAGE_ALL_COMPONENT), new File(ADD_ON_PACKAGE_ALL_COMPONENT_PKG_FILE)) def clusterpath = buildClusterPath(CLUSTER) File launchReportFile = createTempJsonFile(); - cleanupHdfsFile(TARGET_FILE) - + SliderShell shell = createTemplatedSliderApplication(CLUSTER, APP_TEMPLATE, APP_RESOURCE2, @@ -163,8 +172,7 @@ public class ApplicationWithAddonPackagesIT extends AgentCommandTestBase{ launchReportFile) logShell(shell) - Thread.sleep(10000); - def appId = ensureYarnApplicationIsUp(launchReportFile) + ensureYarnApplicationIsUp(launchReportFile) exists(0, CLUSTER) list(0, [CLUSTER]) @@ -174,21 +182,17 @@ public class ApplicationWithAddonPackagesIT extends AgentCommandTestBase{ list(0, [ARG_LIVE]) list(0, [ARG_STATE, "running"]) status(0, CLUSTER) - - Thread.sleep(10000) - verifyFileExist(TARGET_FILE) + awaitTargetFileExists() } - - + @Test - public void testCreateApplicationWithMultipeAddonPackages() throws Throwable { + public void testCreateApplicationWithMultipleAddonPackages() throws Throwable { describe("Create a cluster with multiple addon packages") SliderUtils.zipFolder(new File(ADD_ON_PACKAGE_ALL_COMPONENT), new File(ADD_ON_PACKAGE_ALL_COMPONENT_PKG_FILE)) SliderUtils.zipFolder(new File(ADD_ON_PACKAGE_ONE_COMPONENT), new File(ADD_ON_PACKAGE_ONE_COMPONENT_PKG_FILE)) def clusterpath = buildClusterPath(CLUSTER) File launchReportFile = createTempJsonFile(); - cleanupHdfsFile(TARGET_FILE) - + SliderShell shell = createTemplatedSliderApplication(CLUSTER, APP_TEMPLATE, APP_RESOURCE2, @@ -208,8 +212,19 @@ public class ApplicationWithAddonPackagesIT extends AgentCommandTestBase{ list(0, [ARG_LIVE]) list(0, [ARG_STATE, "running"]) status(0, CLUSTER) - Thread.sleep(10000) - verifyFileExist(TARGET_FILE) + + awaitTargetFileExists() } + + /** + * Here to check file permissions and isolate failures due to that alone + * @throws Throwable + */ + @Test + public void testCreateTargetFile() throws Throwable { + ContractTestUtils.touch(clusterFS, new Path(TARGET_FILE)) + awaitTargetFileExists() + } + }