Merge branch 'master' into sijie/bookkeeper_fallocate
Project: http://git-wip-us.apache.org/repos/asf/bookkeeper/repo Commit: http://git-wip-us.apache.org/repos/asf/bookkeeper/commit/42922828 Tree: http://git-wip-us.apache.org/repos/asf/bookkeeper/tree/42922828 Diff: http://git-wip-us.apache.org/repos/asf/bookkeeper/diff/42922828 Branch: refs/heads/sijie/bookkeeper_fallocate Commit: 42922828e48e9541bb2ad7fdc6d98966ead1c7c3 Parents: 6077ef8 de59bd2 Author: Sijie Guo <[email protected]> Authored: Tue Apr 18 11:02:30 2017 -0700 Committer: Sijie Guo <[email protected]> Committed: Tue Apr 18 11:02:30 2017 -0700 ---------------------------------------------------------------------- bookkeeper-benchmark/pom.xml | 6 + bookkeeper-server/conf/bk_server.conf | 14 + bookkeeper-server/pom.xml | 13 +- .../apache/bookkeeper/auth/AuthCallbacks.java | 28 + .../auth/AuthProviderFactoryFactory.java | 52 +- .../org/apache/bookkeeper/auth/AuthToken.java | 44 + .../bookkeeper/auth/BookKeeperPrincipal.java | 72 + .../bookkeeper/auth/BookieAuthProvider.java | 26 +- .../bookkeeper/auth/ClientAuthProvider.java | 29 +- .../bookie/BookKeeperServerStats.java | 3 + .../org/apache/bookkeeper/bookie/Bookie.java | 447 +- .../apache/bookkeeper/bookie/BookieBean.java | 7 +- .../bookkeeper/bookie/BookieConnectionPeer.java | 30 + .../apache/bookkeeper/bookie/BookieShell.java | 573 +- .../bookkeeper/bookie/CheckpointSourceList.java | 112 + .../org/apache/bookkeeper/bookie/Cookie.java | 106 +- .../apache/bookkeeper/bookie/EntryLogger.java | 33 +- .../apache/bookkeeper/bookie/EntryMemTable.java | 33 +- .../org/apache/bookkeeper/bookie/FileInfo.java | 34 + .../bookkeeper/bookie/FileSystemUpgrade.java | 3 +- .../bookie/GarbageCollectorThread.java | 138 +- .../bookkeeper/bookie/HandleFactoryImpl.java | 55 +- .../bookkeeper/bookie/IndexPersistenceMgr.java | 28 + .../bookie/InterleavedLedgerStorage.java | 21 + .../org/apache/bookkeeper/bookie/Journal.java | 13 +- .../apache/bookkeeper/bookie/LedgerCache.java | 3 + .../bookkeeper/bookie/LedgerCacheImpl.java | 9 + .../bookkeeper/bookie/LedgerDescriptor.java | 5 + .../bookkeeper/bookie/LedgerDescriptorImpl.java | 9 + .../bookkeeper/bookie/LedgerDirsManager.java | 72 + .../apache/bookkeeper/bookie/LedgerStorage.java | 16 + .../LocalBookieEnsemblePlacementPolicy.java | 12 +- .../bookie/ScanAndCompareGarbageCollector.java | 6 +- .../bookkeeper/bookie/SortedLedgerStorage.java | 2 +- .../apache/bookkeeper/client/AsyncCallback.java | 14 + .../apache/bookkeeper/client/BKException.java | 2 +- .../apache/bookkeeper/client/BookKeeper.java | 174 +- .../bookkeeper/client/BookKeeperAdmin.java | 22 +- .../client/BookKeeperClientStats.java | 8 + .../bookkeeper/client/BookieInfoReader.java | 261 + .../apache/bookkeeper/client/BookieWatcher.java | 11 +- .../bookkeeper/client/ClientConnectionPeer.java | 30 + .../client/DefaultEnsemblePlacementPolicy.java | 112 +- .../apache/bookkeeper/client/DigestManager.java | 55 + .../client/EnsemblePlacementPolicy.java | 10 + .../client/ExplicitLacFlushPolicy.java | 152 + .../bookkeeper/client/LedgerCreateOp.java | 26 +- .../apache/bookkeeper/client/LedgerHandle.java | 142 +- .../bookkeeper/client/PendingReadLacOp.java | 145 + .../bookkeeper/client/PendingWriteLacOp.java | 114 + .../RackawareEnsemblePlacementPolicy.java | 15 +- .../RackawareEnsemblePlacementPolicyImpl.java | 177 +- .../bookkeeper/client/ReadOnlyLedgerHandle.java | 12 +- .../RegionAwareEnsemblePlacementPolicy.java | 7 +- .../bookkeeper/client/SynchCallbackUtils.java | 14 +- .../client/WeightedRandomSelection.java | 156 + .../bookkeeper/conf/AbstractConfiguration.java | 75 + .../bookkeeper/conf/ClientConfiguration.java | 182 +- .../bookkeeper/conf/ServerConfiguration.java | 106 +- .../meta/AbstractZkLedgerManager.java | 8 +- .../meta/FlatLedgerManagerFactory.java | 5 +- .../meta/HierarchicalLedgerManager.java | 4 +- .../meta/HierarchicalLedgerManagerFactory.java | 5 +- .../apache/bookkeeper/meta/LedgerLayout.java | 6 +- .../bookkeeper/meta/LedgerManagerFactory.java | 6 +- .../meta/LongHierarchicalLedgerManager.java | 334 + .../LongHierarchicalLedgerManagerFactory.java | 29 + .../bookkeeper/meta/MSLedgerManagerFactory.java | 6 +- .../bookkeeper/meta/ZkLedgerIdGenerator.java | 9 +- .../meta/ZkLedgerUnderreplicationManager.java | 28 +- .../apache/bookkeeper/proto/AuthHandler.java | 147 +- .../apache/bookkeeper/proto/BookieClient.java | 111 +- .../bookkeeper/proto/BookieNettyServer.java | 83 +- .../apache/bookkeeper/proto/BookieProtocol.java | 3 + .../proto/BookieRequestProcessor.java | 65 + .../apache/bookkeeper/proto/BookieServer.java | 83 +- .../proto/BookkeeperInternalCallbacks.java | 13 + .../bookkeeper/proto/BookkeeperProtocol.java | 8003 ++++++++++++++---- .../apache/bookkeeper/proto/ChannelManager.java | 14 +- .../apache/bookkeeper/proto/ConnectionPeer.java | 63 + .../proto/GetBookieInfoProcessorV3.java | 90 + .../bookkeeper/proto/LocalBookiesRegistry.java | 4 +- .../proto/NioServerSocketChannelManager.java | 14 +- .../proto/PerChannelBookieClient.java | 841 +- .../bookkeeper/proto/ReadLacProcessorV3.java | 108 + .../bookkeeper/proto/VMLocalChannelManager.java | 14 +- .../bookkeeper/proto/WriteLacProcessorV3.java | 113 + .../apache/bookkeeper/replication/Auditor.java | 5 +- .../bookkeeper/replication/AuditorElector.java | 10 +- .../replication/ReplicationWorker.java | 110 +- .../bookkeeper/util/HexDumpEntryFormatter.java | 53 + .../apache/bookkeeper/util/LocalBookKeeper.java | 1 + .../bookkeeper/util/OrderedSafeExecutor.java | 117 +- .../org/apache/bookkeeper/util/StringUtils.java | 49 + .../org/apache/bookkeeper/util/ZkUtils.java | 11 + .../util/collections/ConcurrentLongHashMap.java | 494 ++ .../util/collections/ConcurrentLongHashSet.java | 421 + .../collections/ConcurrentLongLongHashMap.java | 723 ++ .../ConcurrentLongLongPairHashMap.java | 550 ++ .../util/collections/ConcurrentOpenHashMap.java | 493 ++ .../util/collections/ConcurrentOpenHashSet.java | 416 + .../bookkeeper/zookeeper/ZooKeeperClient.java | 235 +- .../src/main/proto/BookkeeperProtocol.proto | 63 +- .../src/main/resources/findbugsExclude.xml | 9 + .../org/apache/bookkeeper/auth/TestAuth.java | 381 +- .../bookkeeper/bookie/BookieAccessor.java | 6 +- .../bookkeeper/bookie/CompactionTest.java | 13 +- .../apache/bookkeeper/bookie/CookieTest.java | 145 +- .../bookkeeper/bookie/CreateNewLogTest.java | 32 +- .../bookie/EnableZkSecurityBasicTest.java | 126 + .../bookkeeper/bookie/LedgerCacheTest.java | 89 + .../bookkeeper/bookie/LedgerStorageTest.java | 53 + .../bookie/TestGcOverreplicatedLedger.java | 3 +- .../bookie/TestLedgerDirsManager.java | 30 + .../bookkeeper/bookie/TestSyncThread.java | 13 + .../bookkeeper/bookie/UpdateCookieCmdTest.java | 8 +- .../bookkeeper/client/BookKeeperCloseTest.java | 103 +- ...perDiskSpaceWeightedLedgerPlacementTest.java | 452 + .../bookkeeper/client/BookKeeperTest.java | 124 + .../bookkeeper/client/BookieRecoveryTest.java | 1 + .../client/BookieWriteLedgerTest.java | 103 +- .../client/TestGetBookieInfoTimeout.java | 141 + .../TestRackawareEnsemblePlacementPolicy.java | 248 + ...awareEnsemblePlacementPolicyUsingScript.java | 291 + .../client/TestWatchEnsembleChange.java | 4 +- .../client/TestWeightedRandomSelection.java | 280 + .../NoSystemPropertiesConfigurationTest.java | 14 +- .../conf/SystemPropertiesConfigurationTest.java | 14 +- .../apache/bookkeeper/meta/GcLedgersTest.java | 126 +- .../bookkeeper/meta/LedgerLayoutTest.java | 5 +- .../bookkeeper/meta/LedgerManagerTestCase.java | 20 +- .../bookkeeper/meta/TestLedgerManager.java | 7 +- .../meta/TestZkLedgerIdGenerator.java | 3 +- .../proto/TestBackwardCompatCMS42.java | 45 +- .../bookkeeper/proto/TestDataFormats.java | 149 - .../proto/TestPerChannelBookieClient.java | 2 +- .../replication/AuditorLedgerCheckerTest.java | 3 + .../replication/AuthAutoRecoveryTest.java | 111 + .../bookkeeper/test/BookieClientTest.java | 66 +- .../test/LocalBookiesRegistryTest.java | 59 + .../MultiLedgerManagerMultiDigestTestCase.java | 1 + .../test/MultiLedgerManagerTestCase.java | 3 +- .../apache/bookkeeper/util/TestDiskChecker.java | 33 +- .../collections/ConcurrentLongHashMapTest.java | 435 + .../collections/ConcurrentLongHashSetTest.java | 275 + .../ConcurrentLongLongHashMapTest.java | 473 ++ .../ConcurrentLongLongPairHashMapTest.java | 343 + .../collections/ConcurrentOpenHashMapTest.java | 488 ++ .../collections/ConcurrentOpenHashSetTest.java | 318 + .../bookkeeper/zookeeper/TestRetryPolicy.java | 54 + .../zookeeper/TestZooKeeperClient.java | 29 +- .../src/test/proto/TestDataFormats.proto | 34 - .../src/test/resources/networkmappingscript.sh | 35 + .../codahale-metrics-provider/pom.xml | 16 +- .../stats/CodahaleMetricsProvider.java | 4 +- dev/bk-merge-pr.py | 33 +- pom.xml | 2 +- 157 files changed, 21276 insertions(+), 2615 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/bookkeeper/blob/42922828/bookkeeper-server/pom.xml ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/bookkeeper/blob/42922828/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/BookKeeperServerStats.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/bookkeeper/blob/42922828/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/Journal.java ---------------------------------------------------------------------- diff --cc bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/Journal.java index 942a93f,1483e36..e3531f6 --- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/Journal.java +++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/Journal.java @@@ -819,9 -787,10 +823,10 @@@ class Journal extends BookieCriticalThr */ @Override public void run() { + LOG.info("Starting journal on {}", journalDirectory); LinkedList<QueueEntry> toFlush = new LinkedList<QueueEntry>(); ByteBuffer lenBuff = ByteBuffer.allocate(4); - ByteBuffer paddingBuff = ByteBuffer.allocate(2 * conf.getJournalAlignmentSize()); + ByteBuffer paddingBuff = ByteBuffer.allocate(2 * journalAlignmentSize); ZeroBuffer.put(paddingBuff); JournalChannel logFile = null; forceWriteThread.start();
