[01/34] asterixdb git commit: [NO ISSUE][RT] Ensure all NC tasks are aborted before joining
Repository: asterixdb Updated Branches: refs/heads/release-0.9.4-pre-rc e6587f628 -> 2a773feef [NO ISSUE][RT] Ensure all NC tasks are aborted before joining - user model changes: no - storage format changes: no - interface changes: yes - Add getApplication() to NodeControllerService details: - This change ensures that all previous tasks of a CC on an NC are completed before completion of registration. Change-Id: I0517e5a390d50e8703ffdbecbb84467c22edda85 Reviewed-on: https://asterix-gerrit.ics.uci.edu/2507 Tested-by: Jenkins Contrib: Jenkins Reviewed-by: Michael Blow Integration-Tests: Jenkins Project: http://git-wip-us.apache.org/repos/asf/asterixdb/repo Commit: http://git-wip-us.apache.org/repos/asf/asterixdb/commit/a77ec691 Tree: http://git-wip-us.apache.org/repos/asf/asterixdb/tree/a77ec691 Diff: http://git-wip-us.apache.org/repos/asf/asterixdb/diff/a77ec691 Branch: refs/heads/release-0.9.4-pre-rc Commit: a77ec691d466fab8454b883d7e004074c6c9cf84 Parents: e6587f6 Author: Abdullah Alamoudi Authored: Wed Mar 21 14:26:48 2018 -0700 Committer: abdullah alamoudi Committed: Wed Mar 21 18:24:50 2018 -0700 -- .../hyracks/bootstrap/NCApplication.java| 1 + .../hyracks/control/cc/cluster/NodeManager.java | 15 ++- .../control/nc/NodeControllerService.java | 19 ++-- .../org/apache/hyracks/control/nc/Task.java | 17 +++- .../control/nc/work/AbortAllJobsWork.java | 10 +- .../nc/work/EnsureAllCcTasksCompleted.java | 102 +++ .../java/org/apache/hyracks/util/ExitUtil.java | 1 + .../main/java/org/apache/hyracks/util/Span.java | 12 ++- 8 files changed, 150 insertions(+), 27 deletions(-) -- http://git-wip-us.apache.org/repos/asf/asterixdb/blob/a77ec691/asterixdb/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/NCApplication.java -- diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/NCApplication.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/NCApplication.java index 494198b..57d080e 100644 --- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/NCApplication.java +++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/NCApplication.java @@ -211,6 +211,7 @@ public class NCApplication extends BaseNCApplication { ? getCurrentSystemState() : SystemState.HEALTHY; RegistrationTasksRequestMessage.send(ccId, (NodeControllerService) ncServiceCtx.getControllerService(), currentStatus, systemState); +ncs.notifyRegistrationCompleted(ccId); } @Override http://git-wip-us.apache.org/repos/asf/asterixdb/blob/a77ec691/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/cluster/NodeManager.java -- diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/cluster/NodeManager.java b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/cluster/NodeManager.java index e1a36cd..b44a6bb 100644 --- a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/cluster/NodeManager.java +++ b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/cluster/NodeManager.java @@ -103,14 +103,13 @@ public class NodeManager implements INodeManager { if (nodeRegistry.containsKey(nodeId)) { LOGGER.warn("Node with name " + nodeId + " has already registered; failing the node then re-registering."); failNode(nodeId); -} else { -try { -// TODO(mblow): it seems we should close IPC handles when we're done with them (like here) -IIPCHandle ncIPCHandle = ccs.getClusterIPC().getHandle(ncState.getNodeController().getAddress()); -ncIPCHandle.send(-1, new AbortCCJobsFunction(ccConfig.getCcId()), null); -} catch (IPCException e) { -throw HyracksDataException.create(e); -} +} +try { +// TODO(mblow): it seems we should close IPC handles when we're done with them (like here) +IIPCHandle ncIPCHandle = ccs.getClusterIPC().getHandle(ncState.getNodeController().getAddress()); +ncIPCHandle.send(-1, new AbortCCJobsFunction(ccConfig.getCcId()), null); +} catch (IPCException e) { +throw HyracksDataException.create(e); } LOGGER.warn("adding node to registry"); nodeRegistry.put(nodeId, ncState); http://git-wip-us.apache.org/repos/asf/a
[02/34] asterixdb git commit: [NO ISSUE][TEST] Add timeout multiplier for test executor timeouts
[NO ISSUE][TEST] Add timeout multiplier for test executor timeouts Change-Id: I340098ce840f31f2afcef5dd66e43a21ed7ceb9d Reviewed-on: https://asterix-gerrit.ics.uci.edu/2513 Reviewed-by: Murtadha Hubail Tested-by: Michael Blow Project: http://git-wip-us.apache.org/repos/asf/asterixdb/repo Commit: http://git-wip-us.apache.org/repos/asf/asterixdb/commit/2587f1db Tree: http://git-wip-us.apache.org/repos/asf/asterixdb/tree/2587f1db Diff: http://git-wip-us.apache.org/repos/asf/asterixdb/diff/2587f1db Branch: refs/heads/release-0.9.4-pre-rc Commit: 2587f1dbe58f7fa65163d141eed1c40aee3fdb1a Parents: a77ec69 Author: Michael Blow Authored: Thu Mar 22 17:09:16 2018 -0400 Committer: Michael Blow Committed: Thu Mar 22 17:22:58 2018 -0700 -- .../org/apache/asterix/test/common/TestExecutor.java | 14 ++ 1 file changed, 10 insertions(+), 4 deletions(-) -- http://git-wip-us.apache.org/repos/asf/asterixdb/blob/2587f1db/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/TestExecutor.java -- diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/TestExecutor.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/TestExecutor.java index 8c0a3d4..80048bd 100644 --- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/TestExecutor.java +++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/TestExecutor.java @@ -154,6 +154,7 @@ public class TestExecutor { protected int endpointSelector; protected IExternalUDFLibrarian librarian; private Map testLoops = new HashMap<>(); +private double timeoutMultiplier = 1; public TestExecutor() { this(Inet4Address.getLoopbackAddress().getHostAddress(), 19002); @@ -183,6 +184,10 @@ public class TestExecutor { this.replicationAddress = replicationAddress; } +public void setTimeoutMultiplier(double timeoutMultiplier) { +this.timeoutMultiplier = timeoutMultiplier; +} + /** * Probably does not work well with symlinks. */ @@ -1449,10 +1454,10 @@ public class TestExecutor { return false; } -public static int getTimeoutSecs(String statement) { +public int getTimeoutSecs(String statement) { final Matcher timeoutMatcher = POLL_TIMEOUT_PATTERN.matcher(statement); if (timeoutMatcher.find()) { -return Integer.parseInt(timeoutMatcher.group(1)); +return (int) (Integer.parseInt(timeoutMatcher.group(1)) * timeoutMultiplier); } else { throw new IllegalArgumentException("ERROR: polltimeoutsecs=nnn must be present in poll file"); } @@ -1796,7 +1801,8 @@ public class TestExecutor { waitForClusterState("ACTIVE", timeoutSecs, timeUnit); } -public void waitForClusterState(String desiredState, int timeout, TimeUnit timeUnit) throws Exception { +public void waitForClusterState(String desiredState, int baseTimeout, TimeUnit timeUnit) throws Exception { +int timeout = (int) (baseTimeout * timeoutMultiplier); LOGGER.info("Waiting for cluster state " + desiredState + "..."); Thread t = new Thread(() -> { while (true) { @@ -1874,7 +1880,7 @@ public class TestExecutor { } String host = command[0]; int port = Integer.parseInt(command[1]); -int timeoutSec = Integer.parseInt(command[2]); +int timeoutSec = (int) (Integer.parseInt(command[2]) * timeoutMultiplier); while (isPortActive(host, port)) { TimeUnit.SECONDS.sleep(1); timeoutSec--;
[06/34] asterixdb git commit: [NO ISSUE][CLUS] Complete NC Registration When Response is Received
[NO ISSUE][CLUS] Complete NC Registration When Response is Received - user model changes: no - storage format changes: no - interface changes: no Details: - Mark NC registration as completed when the regisration reponse is received from CC. - Send NC startup tasks request to CC after all previous tasks complete. Change-Id: I4ff41f86a11b52cae894fe40ffa0353f2fb52138 Reviewed-on: https://asterix-gerrit.ics.uci.edu/2525 Reviewed-by: Michael Blow Tested-by: Murtadha Hubail Project: http://git-wip-us.apache.org/repos/asf/asterixdb/repo Commit: http://git-wip-us.apache.org/repos/asf/asterixdb/commit/042353dc Tree: http://git-wip-us.apache.org/repos/asf/asterixdb/tree/042353dc Diff: http://git-wip-us.apache.org/repos/asf/asterixdb/diff/042353dc Branch: refs/heads/release-0.9.4-pre-rc Commit: 042353dc8d4fa971bef86438776a40d1bac2e900 Parents: d71cb24 Author: Murtadha Hubail Authored: Mon Mar 26 09:09:27 2018 +0300 Committer: Murtadha Hubail Committed: Mon Mar 26 09:20:13 2018 -0700 -- .../hyracks/bootstrap/NCApplication.java| 1 - .../apache/hyracks/api/util/ExceptionUtils.java | 12 .../control/nc/NodeControllerService.java | 12 ++-- .../nc/work/EnsureAllCcTasksCompleted.java | 72 +--- 4 files changed, 51 insertions(+), 46 deletions(-) -- http://git-wip-us.apache.org/repos/asf/asterixdb/blob/042353dc/asterixdb/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/NCApplication.java -- diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/NCApplication.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/NCApplication.java index 57d080e..494198b 100644 --- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/NCApplication.java +++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/NCApplication.java @@ -211,7 +211,6 @@ public class NCApplication extends BaseNCApplication { ? getCurrentSystemState() : SystemState.HEALTHY; RegistrationTasksRequestMessage.send(ccId, (NodeControllerService) ncServiceCtx.getControllerService(), currentStatus, systemState); -ncs.notifyRegistrationCompleted(ccId); } @Override http://git-wip-us.apache.org/repos/asf/asterixdb/blob/042353dc/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/util/ExceptionUtils.java -- diff --git a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/util/ExceptionUtils.java b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/util/ExceptionUtils.java index 444b08f..9302f46 100644 --- a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/util/ExceptionUtils.java +++ b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/util/ExceptionUtils.java @@ -107,4 +107,16 @@ public class ExceptionUtils { } return first; } + +/** + * Returns a throwable containing {@code thread} stacktrace + * + * @param thread + * @return The throwable with {@code thread} stacktrace + */ +public static Throwable fromThreadStack(Thread thread) { +final Throwable stackThrowable = new Throwable(thread.getName() + " Stack trace"); +stackThrowable.setStackTrace(thread.getStackTrace()); +return stackThrowable; +} } http://git-wip-us.apache.org/repos/asf/asterixdb/blob/042353dc/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/NodeControllerService.java -- diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/NodeControllerService.java b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/NodeControllerService.java index 6d54843..a74a1ab 100644 --- a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/NodeControllerService.java +++ b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/NodeControllerService.java @@ -441,6 +441,8 @@ public class NodeControllerService implements IControllerService { ccTimer.schedule(new ProfileDumpTask(ccs, ccId), 0, nodeParameters.getProfileDumpPeriod()); ccTimers.put(ccId, ccTimer); } +ccc.notifyRegistrationCompleted(); +LOGGER.info("Registering with Cluster Controller {} completed", ccc); return ccId; } @@ -661,6 +663,10 @@ public class NodeCo
[13/34] asterixdb git commit: [NO ISSUE] Incremental cleanup of deprecated exception ctors
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/6453ecf2/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/partitions/ReceiveSideMaterializingCollector.java -- diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/partitions/ReceiveSideMaterializingCollector.java b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/partitions/ReceiveSideMaterializingCollector.java index b8bdda7..c902ad8 100644 --- a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/partitions/ReceiveSideMaterializingCollector.java +++ b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/partitions/ReceiveSideMaterializingCollector.java @@ -149,7 +149,7 @@ public class ReceiveSideMaterializingCollector implements IPartitionCollector { } } } catch (InterruptedException e) { -throw new HyracksDataException(e); +throw HyracksDataException.create(e); } } } http://git-wip-us.apache.org/repos/asf/asterixdb/blob/6453ecf2/hyracks-fullstack/hyracks/hyracks-dataflow-common/src/main/java/org/apache/hyracks/dataflow/common/data/marshalling/ByteArraySerializerDeserializer.java -- diff --git a/hyracks-fullstack/hyracks/hyracks-dataflow-common/src/main/java/org/apache/hyracks/dataflow/common/data/marshalling/ByteArraySerializerDeserializer.java b/hyracks-fullstack/hyracks/hyracks-dataflow-common/src/main/java/org/apache/hyracks/dataflow/common/data/marshalling/ByteArraySerializerDeserializer.java index ad3ede5..9487681 100644 --- a/hyracks-fullstack/hyracks/hyracks-dataflow-common/src/main/java/org/apache/hyracks/dataflow/common/data/marshalling/ByteArraySerializerDeserializer.java +++ b/hyracks-fullstack/hyracks/hyracks-dataflow-common/src/main/java/org/apache/hyracks/dataflow/common/data/marshalling/ByteArraySerializerDeserializer.java @@ -77,7 +77,7 @@ public class ByteArraySerializerDeserializer implements ISerializerDeserializer< try { out.write(byteArrayPtr.getByteArray(), byteArrayPtr.getStartOffset(), byteArrayPtr.getLength()); } catch (IOException e) { -throw new HyracksDataException(e); +throw HyracksDataException.create(e); } } @@ -89,7 +89,7 @@ public class ByteArraySerializerDeserializer implements ISerializerDeserializer< out.write(metaBuffer, 0, metaLength); out.write(instance, start, length); } catch (IOException e) { -throw new HyracksDataException(e); +throw HyracksDataException.create(e); } } http://git-wip-us.apache.org/repos/asf/asterixdb/blob/6453ecf2/hyracks-fullstack/hyracks/hyracks-dataflow-common/src/main/java/org/apache/hyracks/dataflow/common/data/parsers/ByteArrayBase64ParserFactory.java -- diff --git a/hyracks-fullstack/hyracks/hyracks-dataflow-common/src/main/java/org/apache/hyracks/dataflow/common/data/parsers/ByteArrayBase64ParserFactory.java b/hyracks-fullstack/hyracks/hyracks-dataflow-common/src/main/java/org/apache/hyracks/dataflow/common/data/parsers/ByteArrayBase64ParserFactory.java index 91fb2eb..69db7f3 100644 --- a/hyracks-fullstack/hyracks/hyracks-dataflow-common/src/main/java/org/apache/hyracks/dataflow/common/data/parsers/ByteArrayBase64ParserFactory.java +++ b/hyracks-fullstack/hyracks/hyracks-dataflow-common/src/main/java/org/apache/hyracks/dataflow/common/data/parsers/ByteArrayBase64ParserFactory.java @@ -47,7 +47,7 @@ public class ByteArrayBase64ParserFactory implements IValueParserFactory { try { serializer.serialize(parser.getByteArray(), 0, parser.getLength(), out); } catch (IOException e) { -throw new HyracksDataException(e); +throw HyracksDataException.create(e); } } }; http://git-wip-us.apache.org/repos/asf/asterixdb/blob/6453ecf2/hyracks-fullstack/hyracks/hyracks-dataflow-common/src/main/java/org/apache/hyracks/dataflow/common/data/parsers/ByteArrayHexParserFactory.java -- diff --git a/hyracks-fullstack/hyracks/hyracks-dataflow-common/src/main/java/org/apache/hyracks/dataflow/common/data/parsers/ByteArrayHexParserFactory.java b/hyracks-fullstack/hyracks/hyracks-dataflow-common/src/main/java/org/apache/hyracks/dataflow/common/data/parsers/ByteArrayHe
[11/34] asterixdb git commit: [ASTERIXDB-2340][FUN] Implement object_length(), object_names()
[ASTERIXDB-2340][FUN] Implement object_length(), object_names() - user model changes: yes - storage format changes: no - interface changes: no Details: - Implement object_length(), object_names() functions - Consolidate type inferers for some object accessor functions Change-Id: I1108466ffe4f6b002fa7dd25f76dd5e8d537b0be Reviewed-on: https://asterix-gerrit.ics.uci.edu/2522 Tested-by: Jenkins Contrib: Jenkins Integration-Tests: Jenkins Reviewed-by: Till Westmann Project: http://git-wip-us.apache.org/repos/asf/asterixdb/repo Commit: http://git-wip-us.apache.org/repos/asf/asterixdb/commit/f7c7059c Tree: http://git-wip-us.apache.org/repos/asf/asterixdb/tree/f7c7059c Diff: http://git-wip-us.apache.org/repos/asf/asterixdb/diff/f7c7059c Branch: refs/heads/release-0.9.4-pre-rc Commit: f7c7059c7e3145129d32de474f7401c25df76d32 Parents: 8d28443 Author: Dmitry Lychagin Authored: Mon Mar 26 15:57:48 2018 -0700 Committer: Michael Blow Committed: Mon Mar 26 20:53:58 2018 -0700 -- .../queries_sqlpp/objects/ObjectsQueries.xml| 10 ++ .../object_length/object_length.1.ddl.sqlpp | 49 +++ .../object_length/object_length.2.update.sqlpp | 29 .../object_length/object_length.3.query.sqlpp | 56 .../object_names/object_names.1.ddl.sqlpp | 49 +++ .../object_names/object_names.2.update.sqlpp| 29 .../object_names/object_names.3.query.sqlpp | 59 .../objects/object_length/object_length.3.adm | 1 + .../objects/object_names/object_names.3.adm | 1 + .../src/main/markdown/builtins/8_record.md | 54 .../asterix/om/functions/BuiltinFunctions.java | 6 + .../typecomputer/impl/AInt64TypeComputer.java | 14 +- .../impl/OrderedListOfAStringTypeComputer.java | 13 +- .../records/GetRecordFieldValueDescriptor.java | 2 +- .../records/GetRecordFieldsDescriptor.java | 2 +- .../records/RecordLengthDescriptor.java | 122 + .../records/RecordNamesDescriptor.java | 134 +++ .../records/RecordPairsDescriptor.java | 2 +- .../runtime/functions/FunctionCollection.java | 4 + .../runtime/functions/FunctionTypeInferers.java | 66 - 20 files changed, 653 insertions(+), 49 deletions(-) -- http://git-wip-us.apache.org/repos/asf/asterixdb/blob/f7c7059c/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/objects/ObjectsQueries.xml -- diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/objects/ObjectsQueries.xml b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/objects/ObjectsQueries.xml index c5ff15b..507e276 100644 --- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/objects/ObjectsQueries.xml +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/objects/ObjectsQueries.xml @@ -108,6 +108,16 @@ + + object_length + + + + + object_names + + + object_pairs http://git-wip-us.apache.org/repos/asf/asterixdb/blob/f7c7059c/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/objects/object_length/object_length.1.ddl.sqlpp -- diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/objects/object_length/object_length.1.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/objects/object_length/object_length.1.ddl.sqlpp new file mode 100644 index 000..a50bf21 --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/objects/object_length/object_length.1.ddl.sqlpp @@ -0,0 +1,49 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +/* + * Description : Testing object_length under different queries. + * Expected Res : Success + */ + +drop dataverse TinySocial if exists; +create dataverse TinySocial; + +use TinySocial; + +create type TinySocial.TwitterUserType as +{ + `screen-name` : string
[32/34] asterixdb git commit: [NO ISSUE][RT] Use ThreadLocal date format to format dates
[NO ISSUE][RT] Use ThreadLocal date format to format dates - user model changes: no - storage format changes: no - interface changes: no Details: - Instead of creating a new date format object for formatting trace logs, use a thread local instance. Change-Id: I361542815af7c93291fd6251d9b335c2eb01af56 Reviewed-on: https://asterix-gerrit.ics.uci.edu/2563 Sonar-Qube: Jenkins Tested-by: Jenkins Contrib: Jenkins Integration-Tests: Jenkins Reviewed-by: Michael Blow Project: http://git-wip-us.apache.org/repos/asf/asterixdb/repo Commit: http://git-wip-us.apache.org/repos/asf/asterixdb/commit/3f4cb466 Tree: http://git-wip-us.apache.org/repos/asf/asterixdb/tree/3f4cb466 Diff: http://git-wip-us.apache.org/repos/asf/asterixdb/diff/3f4cb466 Branch: refs/heads/release-0.9.4-pre-rc Commit: 3f4cb466ac08c72e6cc4df927a075a6364559df1 Parents: 29a0cee Author: Abdullah Alamoudi Authored: Thu Apr 5 11:14:43 2018 -0700 Committer: abdullah alamoudi Committed: Thu Apr 5 18:17:47 2018 -0700 -- .../LSMPrimaryUpsertOperatorNodePushable.java | 39 ++-- 1 file changed, 27 insertions(+), 12 deletions(-) -- http://git-wip-us.apache.org/repos/asf/asterixdb/blob/3f4cb466/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/operators/LSMPrimaryUpsertOperatorNodePushable.java -- diff --git a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/operators/LSMPrimaryUpsertOperatorNodePushable.java b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/operators/LSMPrimaryUpsertOperatorNodePushable.java index c767157..dba6760 100644 --- a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/operators/LSMPrimaryUpsertOperatorNodePushable.java +++ b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/operators/LSMPrimaryUpsertOperatorNodePushable.java @@ -21,6 +21,7 @@ package org.apache.asterix.runtime.operators; import java.io.DataOutput; import java.io.IOException; import java.nio.ByteBuffer; +import java.text.DateFormat; import java.text.SimpleDateFormat; import java.util.Date; @@ -73,9 +74,14 @@ import org.apache.hyracks.storage.common.MultiComparator; import org.apache.hyracks.util.trace.ITracer; import org.apache.hyracks.util.trace.ITracer.Scope; import org.apache.hyracks.util.trace.TraceUtils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; public class LSMPrimaryUpsertOperatorNodePushable extends LSMIndexInsertUpdateDeleteOperatorNodePushable { +private static final Logger LOGGER = LogManager.getLogger(); +private static final ThreadLocal DATE_FORMAT = +ThreadLocal.withInitial(() -> new SimpleDateFormat("-MM-dd'T'HH:mm:ss.SSS")); private final PermutingFrameTupleReference key; private MultiComparator keySearchCmp; private ArrayTupleBuilder missingTupleBuilder; @@ -379,22 +385,31 @@ public class LSMPrimaryUpsertOperatorNodePushable extends LSMIndexInsertUpdateDe @Override public void close() throws HyracksDataException { +traceLastRecordIn(); +Throwable failure = CleanupUtils.close(frameOpCallback, null); +failure = CleanupUtils.destroy(failure, cursor); +failure = CleanupUtils.close(writer, failure); +failure = CleanupUtils.close(indexHelper, failure); +if (failure != null) { +throw HyracksDataException.create(failure); +} +} + +@SuppressWarnings({ "squid:S1181", "squid:S1166" }) +private void traceLastRecordIn() { try { -Throwable failure = CleanupUtils.close(frameOpCallback, null); -failure = CleanupUtils.destroy(failure, cursor); -failure = CleanupUtils.close(writer, failure); -failure = CleanupUtils.close(indexHelper, failure); -if (failure != null) { -throw HyracksDataException.create(failure); -} -} finally { -if (tracer.isEnabled(traceCategory) && lastRecordInTimeStamp > 0) { +if (tracer.isEnabled(traceCategory) && lastRecordInTimeStamp > 0 && indexHelper != null +&& indexHelper.getIndexInstance() != null) { tracer.instant("UpsertClose", traceCategory, Scope.t, -"{\"last-record-in\":\"" -+ new SimpleDateFormat("-MM-dd'T'HH:mm:ss.SSS") -.format(new Date(lastRecordInTimeStamp)) +"{\"last-record-in\":\"" + DATE_FORMAT.get().format(new Date(lastRecordInTimeStamp)) + "\", \"index\":" + indexHelper.getIndexInstance().toString() + "}"); } +} catch (Throwable traceFailure) { +try { +
[34/34] asterixdb git commit: [NO ISSUE] Remove dependencies on extraneous xml libraries
[NO ISSUE] Remove dependencies on extraneous xml libraries Change-Id: I31ba64478e63be77994644add78d834642690de8 Reviewed-on: https://asterix-gerrit.ics.uci.edu/2575 Sonar-Qube: Jenkins Tested-by: Jenkins Contrib: Jenkins Integration-Tests: Jenkins Reviewed-by: Till Westmann Project: http://git-wip-us.apache.org/repos/asf/asterixdb/repo Commit: http://git-wip-us.apache.org/repos/asf/asterixdb/commit/2a773fee Tree: http://git-wip-us.apache.org/repos/asf/asterixdb/tree/2a773fee Diff: http://git-wip-us.apache.org/repos/asf/asterixdb/diff/2a773fee Branch: refs/heads/release-0.9.4-pre-rc Commit: 2a773feefd7068f6f3d8e419e28e959255c2494b Parents: fa52d7c Author: Michael Blow Authored: Sun Apr 8 13:11:26 2018 -0400 Committer: Michael Blow Committed: Sun Apr 8 11:50:42 2018 -0700 -- asterixdb/asterix-external-data/pom.xml | 19 --- asterixdb/pom.xml | 8 2 files changed, 8 insertions(+), 19 deletions(-) -- http://git-wip-us.apache.org/repos/asf/asterixdb/blob/2a773fee/asterixdb/asterix-external-data/pom.xml -- diff --git a/asterixdb/asterix-external-data/pom.xml b/asterixdb/asterix-external-data/pom.xml index 521e176..78549b3 100644 --- a/asterixdb/asterix-external-data/pom.xml +++ b/asterixdb/asterix-external-data/pom.xml @@ -131,20 +131,6 @@ -org.apache.maven.plugins -maven-dependency-plugin - - - org.json:json:* - stax:stax-api:* - javax.xml.bind:jaxb-api:* - - - xml-apis:xml-apis:* - - - - org.apache.rat apache-rat-plugin @@ -395,11 +381,6 @@ hyracks-api - xml-apis - xml-apis - 1.4.01 - - com.fasterxml.jackson.core jackson-databind http://git-wip-us.apache.org/repos/asf/asterixdb/blob/2a773fee/asterixdb/pom.xml -- diff --git a/asterixdb/pom.xml b/asterixdb/pom.xml index b555cde..d1aed7c 100644 --- a/asterixdb/pom.xml +++ b/asterixdb/pom.xml @@ -862,6 +862,14 @@ commons-logging commons-logging + +stax +stax-api + + +javax.xml.bind +jaxb-api +
[09/34] asterixdb git commit: [NO ISSUE][RT] Add job start timestamp to the joblet context
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/8d284433/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/temporal/PrintDateDescriptor.java -- diff --git a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/temporal/PrintDateDescriptor.java b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/temporal/PrintDateDescriptor.java index 1ddb7e8..2879576 100644 --- a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/temporal/PrintDateDescriptor.java +++ b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/temporal/PrintDateDescriptor.java @@ -23,14 +23,14 @@ import java.io.IOException; import org.apache.asterix.dataflow.data.nontagged.serde.ADateSerializerDeserializer; import org.apache.asterix.om.base.temporal.DateTimeFormatUtils; -import org.apache.asterix.om.base.temporal.GregorianCalendarSystem; import org.apache.asterix.om.base.temporal.DateTimeFormatUtils.DateTimeParseMode; -import org.apache.asterix.runtime.exceptions.TypeMismatchException; +import org.apache.asterix.om.base.temporal.GregorianCalendarSystem; import org.apache.asterix.om.functions.BuiltinFunctions; import org.apache.asterix.om.functions.IFunctionDescriptor; import org.apache.asterix.om.functions.IFunctionDescriptorFactory; import org.apache.asterix.om.types.ATypeTag; import org.apache.asterix.runtime.evaluators.base.AbstractScalarFunctionDynamicDescriptor; +import org.apache.asterix.runtime.exceptions.TypeMismatchException; import org.apache.hyracks.algebricks.core.algebra.functions.FunctionIdentifier; import org.apache.hyracks.algebricks.runtime.base.IScalarEvaluator; import org.apache.hyracks.algebricks.runtime.base.IScalarEvaluatorFactory; @@ -45,8 +45,6 @@ import org.apache.hyracks.util.string.UTF8StringWriter; public class PrintDateDescriptor extends AbstractScalarFunctionDynamicDescriptor { private static final long serialVersionUID = 1L; -public final static FunctionIdentifier FID = BuiltinFunctions.PRINT_DATE; -private final static DateTimeFormatUtils DT_UTILS = DateTimeFormatUtils.getInstance(); public final static IFunctionDescriptorFactory FACTORY = new IFunctionDescriptorFactory() { @@ -76,6 +74,8 @@ public class PrintDateDescriptor extends AbstractScalarFunctionDynamicDescriptor private StringBuilder sbder = new StringBuilder(); private final UTF8StringWriter utf8Writer = new UTF8StringWriter(); +private final DateTimeFormatUtils util = DateTimeFormatUtils.getInstance(); + @Override public void evaluate(IFrameTupleReference tuple, IPointable result) throws HyracksDataException { resultStorage.reset(); @@ -101,11 +101,11 @@ public class PrintDateDescriptor extends AbstractScalarFunctionDynamicDescriptor int formatLength = UTF8StringUtil.getUTFLength(bytes1, offset1 + 1); int offset = UTF8StringUtil.getNumBytesToStoreLength(formatLength); sbder.delete(0, sbder.length()); -DT_UTILS.printDateTime(chronon, 0, bytes1, offset1 + 1 + offset, formatLength, sbder, +util.printDateTime(chronon, 0, bytes1, offset1 + 1 + offset, formatLength, sbder, DateTimeParseMode.DATE_ONLY); out.writeByte(ATypeTag.SERIALIZED_STRING_TYPE_TAG); -utf8Writer.writeUTF8(sbder.toString(), out); +utf8Writer.writeUTF8(sbder, out); } catch (IOException ex) { throw new HyracksDataException(ex); } @@ -113,7 +113,6 @@ public class PrintDateDescriptor extends AbstractScalarFunctionDynamicDescriptor } }; } - }; } @@ -122,7 +121,6 @@ public class PrintDateDescriptor extends AbstractScalarFunctionDynamicDescriptor */ @Override public FunctionIdentifier getIdentifier() { -return FID; +return BuiltinFunctions.PRINT_DATE; } - } http://git-wip-us.apache.org/repos/asf/asterixdb/blob/8d284433/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/temporal/PrintDateTimeDescriptor.java -- diff --git a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/temporal/PrintDateTimeDescriptor.java b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/temporal/PrintDateTimeDescriptor.java index b9f6e50..6f22bf1 100644 --- a/a
[12/34] asterixdb git commit: [NO ISSUE][REPL] Ignore LSNs of Partially Replicated Indexes
[NO ISSUE][REPL] Ignore LSNs of Partially Replicated Indexes - user model changes: no - storage format changes: no - interface changes: yes Details: - When determining low watermark, ignore LSN of replicated indexes with no checkpoints. - Guard logs in case of unexpected min LSN read failures. - Ensure only one replica is synchronized at a time to prevent possible merge operations from deleting files being synchronized to another replica concurrently. - Ensure index metadata files are replicated first to allow replicas to find any existing files in case of re-synchronization. - Ensure replication channel is closed on replication failures. Change-Id: I9ca08da29bdd8fc4406f2df7e6eb32601caf9388 Reviewed-on: https://asterix-gerrit.ics.uci.edu/2534 Sonar-Qube: Jenkins Tested-by: Jenkins Contrib: Jenkins Integration-Tests: Jenkins Reviewed-by: Michael Blow Project: http://git-wip-us.apache.org/repos/asf/asterixdb/repo Commit: http://git-wip-us.apache.org/repos/asf/asterixdb/commit/f3784bb3 Tree: http://git-wip-us.apache.org/repos/asf/asterixdb/tree/f3784bb3 Diff: http://git-wip-us.apache.org/repos/asf/asterixdb/diff/f3784bb3 Branch: refs/heads/release-0.9.4-pre-rc Commit: f3784bb3e5c5a62ee244adab80bb70b1b811f255 Parents: f7c7059 Author: Murtadha Hubail Authored: Tue Mar 27 21:28:14 2018 +0300 Committer: Michael Blow Committed: Tue Mar 27 13:53:37 2018 -0700 -- .../apache/asterix/app/nc/RecoveryManager.java | 15 +-- .../apache/asterix/app/nc/ReplicaManager.java| 6 ++ .../asterix/common/storage/IReplicaManager.java | 8 .../replication/api/PartitionReplica.java| 19 +-- .../sync/ReplicaFilesSynchronizer.java | 12 .../replication/sync/ReplicaSynchronizer.java| 13 - 6 files changed, 60 insertions(+), 13 deletions(-) -- http://git-wip-us.apache.org/repos/asf/asterixdb/blob/f3784bb3/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/RecoveryManager.java -- diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/RecoveryManager.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/RecoveryManager.java index 4b14a9c..d4e652d 100644 --- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/RecoveryManager.java +++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/RecoveryManager.java @@ -50,6 +50,7 @@ import org.apache.asterix.common.dataflow.DatasetLocalResource; import org.apache.asterix.common.exceptions.ACIDException; import org.apache.asterix.common.ioopcallbacks.AbstractLSMIOOperationCallback; import org.apache.asterix.common.storage.DatasetResourceReference; +import org.apache.asterix.common.storage.IIndexCheckpointManager; import org.apache.asterix.common.storage.IIndexCheckpointManagerProvider; import org.apache.asterix.common.transactions.Checkpoint; import org.apache.asterix.common.transactions.ICheckpointManager; @@ -93,6 +94,7 @@ import org.apache.logging.log4j.Logger; public class RecoveryManager implements IRecoveryManager, ILifeCycleComponent { public static final boolean IS_DEBUG_MODE = false; +private static final long SMALLEST_POSSIBLE_LSN = 0; private static final Logger LOGGER = org.apache.logging.log4j.LogManager.getLogger(); private final ITransactionSubsystem txnSubsystem; private final LogManager logMgr; @@ -499,8 +501,17 @@ public class RecoveryManager implements IRecoveryManager, ILifeCycleComponent { return dsResource.getPartition() == partition; }).values().stream().map(DatasetResourceReference::of).collect(Collectors.toList()); for (DatasetResourceReference indexRef : partitionResources) { -long remoteIndexMaxLSN = idxCheckpointMgrProvider.get(indexRef).getLowWatermark(); -minRemoteLSN = Math.min(minRemoteLSN, remoteIndexMaxLSN); +try { +final IIndexCheckpointManager idxCheckpointMgr = idxCheckpointMgrProvider.get(indexRef); +if (idxCheckpointMgr.getCheckpointCount() > 0) { +long remoteIndexMaxLSN = idxCheckpointMgrProvider.get(indexRef).getLowWatermark(); +minRemoteLSN = Math.min(minRemoteLSN, remoteIndexMaxLSN); +} +} catch (Exception e) { +LOGGER.warn("Failed to get min LSN of resource {}", indexRef, e); +// ensure no logs will be deleted in case of unexpected failures +return SMALLEST_POSSIBLE_LSN; +} } } return minRemoteLSN; http://git-wip-us.apache.org/repos/asf/asterixdb/blob/f3784bb3/asterixdb/asterix-app/src/main/java/org/apache
[10/34] asterixdb git commit: [NO ISSUE][RT] Add job start timestamp to the joblet context
[NO ISSUE][RT] Add job start timestamp to the joblet context - user model changes: no - storage format changes: no - interface changes: yes Details: - add job start timestamp to the joblet context - make CommonFunctionMapUtil extensible by products - enhance DateTimeFormatUtils.parseDateTime() to return timezone information and not throw exception on invalid data - add GregorianCalendarSystem.getDayOfYear(), getWeekOfYear() - change sleep() to sleep first then evaluate the argument - skip fractional trailing 0s when printing nano-duration - fix documentation about using column aliases in GroupBy Change-Id: I190663ec5e709584ef449f8279b1d2a5a0b099dd Reviewed-on: https://asterix-gerrit.ics.uci.edu/2511 Tested-by: Jenkins Contrib: Jenkins Integration-Tests: Jenkins Reviewed-by: Till Westmann Project: http://git-wip-us.apache.org/repos/asf/asterixdb/repo Commit: http://git-wip-us.apache.org/repos/asf/asterixdb/commit/8d284433 Tree: http://git-wip-us.apache.org/repos/asf/asterixdb/tree/8d284433 Diff: http://git-wip-us.apache.org/repos/asf/asterixdb/diff/8d284433 Branch: refs/heads/release-0.9.4-pre-rc Commit: 8d284433fc4e0e034ce2f032b256b04cb128680f Parents: c4eb7b1 Author: Dmitry Lychagin Authored: Mon Mar 26 16:08:06 2018 -0700 Committer: Michael Blow Committed: Mon Mar 26 20:52:41 2018 -0700 -- .../asterix/runtime/ParseDurationTest.java | 4 +- .../org/apache/asterix/common/api/Duration.java | 39 ++- .../src/main/markdown/sqlpp/3_query.md | 5 +- .../lang/common/util/CommonFunctionMapUtil.java | 96 +++ .../om/base/temporal/DateTimeFormatUtils.java | 266 --- .../temporal/DurationArithmeticOperations.java | 10 +- .../base/temporal/GregorianCalendarSystem.java | 148 +-- ...ervalStartFromDateConstructorDescriptor.java | 5 +- .../evaluators/functions/SleepDescriptor.java | 3 +- .../AdjustDateTimeForTimeZoneDescriptor.java| 2 +- .../AdjustTimeForTimeZoneDescriptor.java| 2 +- .../functions/temporal/DayOfWeekDescriptor.java | 31 +-- .../functions/temporal/ParseDateDescriptor.java | 49 ++-- .../temporal/ParseDateTimeDescriptor.java | 43 ++- .../functions/temporal/ParseTimeDescriptor.java | 43 ++- .../functions/temporal/PrintDateDescriptor.java | 16 +- .../temporal/PrintDateTimeDescriptor.java | 14 +- .../functions/temporal/PrintTimeDescriptor.java | 14 +- .../api/context/IHyracksJobletContext.java | 2 + .../control/cc/executor/JobExecutor.java| 3 +- .../control/common/base/INodeController.java| 2 +- .../control/common/ipc/CCNCFunctions.java | 16 +- .../common/ipc/NodeControllerRemoteProxy.java | 6 +- .../org/apache/hyracks/control/nc/Joblet.java | 10 +- .../hyracks/control/nc/NodeControllerIPCI.java | 2 +- .../hyracks/control/nc/work/StartTasksWork.java | 7 +- .../hyracks/test/support/TestJobletContext.java | 7 + 27 files changed, 521 insertions(+), 324 deletions(-) -- http://git-wip-us.apache.org/repos/asf/asterixdb/blob/8d284433/asterixdb/asterix-app/src/test/java/org/apache/asterix/runtime/ParseDurationTest.java -- diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/runtime/ParseDurationTest.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/runtime/ParseDurationTest.java index f2fb580..d20d72d 100644 --- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/runtime/ParseDurationTest.java +++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/runtime/ParseDurationTest.java @@ -136,7 +136,7 @@ public class ParseDurationTest { public void testDurationFormatNanos() throws Exception { Assert.assertEquals("123.456789012s", Duration.formatNanos(123456789012l)); Assert.assertEquals("12.345678901s", Duration.formatNanos(12345678901l)); -Assert.assertEquals("1.234567890s", Duration.formatNanos(1234567890l)); +Assert.assertEquals("1.23456789s", Duration.formatNanos(1234567890l)); Assert.assertEquals("123.456789ms", Duration.formatNanos(123456789l)); Assert.assertEquals("12.345678ms", Duration.formatNanos(12345678l)); Assert.assertEquals("1.234567ms", Duration.formatNanos(1234567l)); @@ -147,7 +147,7 @@ public class ParseDurationTest { Assert.assertEquals("12ns", Duration.formatNanos(12l)); Assert.assertEquals("1ns", Duration.formatNanos(1l)); Assert.assertEquals("-123.456789012s", Duration.formatNanos(-123456789012l)); -Assert.assertEquals("120.0s", Duration.formatNanos(1200l)); +Assert.assertEquals("120s", Duration.formatNanos(1200l)); Assert.assertEquals("-12ns", Duration.formatNanos(-12l)); } http://git-wip-us.apache.org/repos/asf/asterixdb/blob/8d28443
[23/34] asterixdb git commit: [ASTERIXDB-2343][FUN] Implement to_array(), to_atomic(), to_object()
[ASTERIXDB-2343][FUN] Implement to_array(), to_atomic(), to_object() - user model changes: yes - storage format changes: no - interface changes: no Details: - Implement to_array(), to_atomic(), to_object() functions - Add function aliases: to_bool, to_str Change-Id: I6391a060ecd86cd397dd0a59f4930c4c55216d3b Reviewed-on: https://asterix-gerrit.ics.uci.edu/2539 Reviewed-by: Till Westmann Tested-by: Jenkins Contrib: Jenkins Integration-Tests: Jenkins Project: http://git-wip-us.apache.org/repos/asf/asterixdb/repo Commit: http://git-wip-us.apache.org/repos/asf/asterixdb/commit/09182295 Tree: http://git-wip-us.apache.org/repos/asf/asterixdb/tree/09182295 Diff: http://git-wip-us.apache.org/repos/asf/asterixdb/diff/09182295 Branch: refs/heads/release-0.9.4-pre-rc Commit: 09182295dc03f92a5becd7dd2da93c0838907c3b Parents: cba9ec7 Author: Dmitry Lychagin Authored: Thu Mar 29 15:31:31 2018 -0700 Committer: Till Westmann Committed: Fri Mar 30 00:17:25 2018 -0700 -- .../optimizer/rules/ConstantFoldingRule.java| 3 +- .../types/to_array/to_array.1.query.sqlpp | 33 .../types/to_atomic/to_atomic.1.ddl.sqlpp | 30 +++ .../types/to_atomic/to_atomic.2.update.sqlpp| 32 +++ .../types/to_atomic/to_atomic.3.query.sqlpp | 32 +++ .../types/to_atomic/to_atomic.4.query.sqlpp | 24 +++ .../types/to_object/to_object.1.query.sqlpp | 33 .../to_string_01/to_string_01.1.query.sqlpp | 2 +- .../results/types/to_array/to_array.1.adm | 1 + .../results/types/to_atomic/to_atomic.3.adm | 1 + .../results/types/to_atomic/to_atomic.4.adm | 8 + .../results/types/to_object/to_object.1.adm | 1 + .../resources/runtimets/testsuite_sqlpp.xml | 43 ++-- .../src/main/markdown/builtins/11_type.md | 96 - .../lang/common/util/CommonFunctionMapUtil.java | 11 +- .../asterix/om/functions/BuiltinFunctions.java | 23 ++- .../typecomputer/impl/ToArrayTypeComputer.java | 47 + .../typecomputer/impl/ToObjectTypeComputer.java | 41 .../evaluators/functions/ToArrayDescriptor.java | 117 +++ .../functions/ToAtomicDescriptor.java | 194 +++ .../functions/ToObjectDescriptor.java | 97 ++ .../runtime/functions/FunctionCollection.java | 14 +- .../runtime/functions/FunctionTypeInferers.java | 10 + 23 files changed, 860 insertions(+), 33 deletions(-) -- http://git-wip-us.apache.org/repos/asf/asterixdb/blob/09182295/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/ConstantFoldingRule.java -- diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/ConstantFoldingRule.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/ConstantFoldingRule.java index 7e9328b..29a8e77 100644 --- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/ConstantFoldingRule.java +++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/ConstantFoldingRule.java @@ -92,7 +92,8 @@ public class ConstantFoldingRule implements IAlgebraicRewriteRule { BuiltinFunctions.GET_RECORD_FIELDS, BuiltinFunctions.GET_RECORD_FIELD_VALUE, BuiltinFunctions.FIELD_ACCESS_NESTED, BuiltinFunctions.GET_ITEM, BuiltinFunctions.OPEN_RECORD_CONSTRUCTOR, BuiltinFunctions.FIELD_ACCESS_BY_INDEX, BuiltinFunctions.CAST_TYPE, BuiltinFunctions.META, -BuiltinFunctions.META_KEY, BuiltinFunctions.RECORD_CONCAT, BuiltinFunctions.RECORD_CONCAT_STRICT); +BuiltinFunctions.META_KEY, BuiltinFunctions.RECORD_CONCAT, BuiltinFunctions.RECORD_CONCAT_STRICT, +BuiltinFunctions.TO_ATOMIC, BuiltinFunctions.TO_ARRAY); /** * Throws exceptions in substituiteProducedVariable, setVarType, and one getVarType method. http://git-wip-us.apache.org/repos/asf/asterixdb/blob/09182295/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/types/to_array/to_array.1.query.sqlpp -- diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/types/to_array/to_array.1.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/types/to_array/to_array.1.query.sqlpp new file mode 100644 index 000..c0814b5 --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/types/to_array/to_array.1.query.sqlpp @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License
[05/34] asterixdb git commit: [ASTERIXDB-2216] Disable flaky test which depends on external site
[ASTERIXDB-2216] Disable flaky test which depends on external site The feed http://lorem-rss.herokuapp.com/feed seems to be down, disabling the test which relies on it. Also fixed the assert. Change-Id: Idcf9acf410aa4b8094a93e40681db284a2156345 Reviewed-on: https://asterix-gerrit.ics.uci.edu/2527 Reviewed-by: Michael Blow Tested-by: Michael Blow Project: http://git-wip-us.apache.org/repos/asf/asterixdb/repo Commit: http://git-wip-us.apache.org/repos/asf/asterixdb/commit/d71cb24e Tree: http://git-wip-us.apache.org/repos/asf/asterixdb/tree/d71cb24e Diff: http://git-wip-us.apache.org/repos/asf/asterixdb/diff/d71cb24e Branch: refs/heads/release-0.9.4-pre-rc Commit: d71cb24eb7040875eaf9b70cc5311e87b24f149d Parents: ad1d30a Author: Michael Blow Authored: Mon Mar 26 11:55:35 2018 -0400 Committer: Michael Blow Committed: Mon Mar 26 09:05:00 2018 -0700 -- .../external/input/record/reader/RSSRecordReaderTest.java | 10 +- 1 file changed, 5 insertions(+), 5 deletions(-) -- http://git-wip-us.apache.org/repos/asf/asterixdb/blob/d71cb24e/asterixdb/asterix-external-data/src/test/java/org/apache/asterix/external/input/record/reader/RSSRecordReaderTest.java -- diff --git a/asterixdb/asterix-external-data/src/test/java/org/apache/asterix/external/input/record/reader/RSSRecordReaderTest.java b/asterixdb/asterix-external-data/src/test/java/org/apache/asterix/external/input/record/reader/RSSRecordReaderTest.java index 5033b55..a67b507 100644 --- a/asterixdb/asterix-external-data/src/test/java/org/apache/asterix/external/input/record/reader/RSSRecordReaderTest.java +++ b/asterixdb/asterix-external-data/src/test/java/org/apache/asterix/external/input/record/reader/RSSRecordReaderTest.java @@ -18,14 +18,15 @@ */ package org.apache.asterix.external.input.record.reader; -import com.rometools.rome.feed.synd.SyndEntry; +import java.io.IOException; +import java.net.MalformedURLException; + import org.apache.asterix.external.api.IRawRecord; import org.apache.asterix.external.input.record.reader.rss.RSSRecordReader; import org.junit.Assert; import org.junit.Test; -import java.io.IOException; -import java.net.MalformedURLException; +import com.rometools.rome.feed.synd.SyndEntry; public class RSSRecordReaderTest { @Test @@ -44,7 +45,6 @@ public class RSSRecordReaderTest { private static final int NO_RECORDS = 10; -@Test public void fetchFromLoremWebsite() throws MalformedURLException { String dummyRssFeedURL = "http://lorem-rss.herokuapp.com/feed";; RSSRecordReader rssRecordReader = new RSSRecordReader(dummyRssFeedURL); @@ -59,7 +59,7 @@ public class RSSRecordReaderTest { } catch (Exception e) { expectedException = e; } -Assert.assertEquals(cnt, NO_RECORDS); +Assert.assertEquals(NO_RECORDS, cnt); Assert.assertNull(expectedException); } }
[07/34] asterixdb git commit: [ASTERIXDB-2334] Fix Range Predicate for Composite Key Search
[ASTERIXDB-2334] Fix Range Predicate for Composite Key Search - user model changes: no - storage format changes: no - interface changes: no Details: - Currently during a primary btree search, we always use the range limit of the first search key. However, this is incorrect for prefix range search. This patch fixes this bug by using the correct range limit during index search. Change-Id: Ie2c432c78043913a9dcf9b8a2eb31db8c548d8d2 Reviewed-on: https://asterix-gerrit.ics.uci.edu/2532 Sonar-Qube: Jenkins Contrib: Michael Blow Tested-by: Jenkins Contrib: Jenkins Integration-Tests: Jenkins Reviewed-by: Michael Blow Project: http://git-wip-us.apache.org/repos/asf/asterixdb/repo Commit: http://git-wip-us.apache.org/repos/asf/asterixdb/commit/e541f048 Tree: http://git-wip-us.apache.org/repos/asf/asterixdb/tree/e541f048 Diff: http://git-wip-us.apache.org/repos/asf/asterixdb/diff/e541f048 Branch: refs/heads/release-0.9.4-pre-rc Commit: e541f048f0e5ace442b91d86b647bc22dd7f32ad Parents: 042353d Author: luochen01 Authored: Mon Mar 26 11:24:57 2018 -0700 Committer: Luo Chen Committed: Mon Mar 26 16:36:56 2018 -0700 -- .../optimizer/rules/am/BTreeAccessMethod.java | 17 + .../composite-prefix.1.ddl.sqlpp| 35 ++ .../composite-prefix.2.update.sqlpp | 28 ++ .../composite-prefix.3.query.sqlpp | 27 ++ .../query-ASTERIXDB-2334.1.ddl.sqlpp| 39 .../query-ASTERIXDB-2334.2.update.sqlpp | 28 ++ .../query-ASTERIXDB-920.3.query.sqlpp | 26 + .../composite-low-high/composite-low-high.1.adm | 2 +- .../composite-prefix/composite-prefix.1.adm | 2 + .../query-ASTERIXDB-2334.1.adm | 3 ++ .../resources/runtimets/testsuite_sqlpp.xml | 10 + 11 files changed, 209 insertions(+), 8 deletions(-) -- http://git-wip-us.apache.org/repos/asf/asterixdb/blob/e541f048/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/BTreeAccessMethod.java -- diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/BTreeAccessMethod.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/BTreeAccessMethod.java index a82e780..60d4d3d 100644 --- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/BTreeAccessMethod.java +++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/BTreeAccessMethod.java @@ -528,11 +528,6 @@ public class BTreeAccessMethod implements IAccessMethod { } } -if (primaryIndexPostProccessingIsNeeded) { -Arrays.fill(lowKeyInclusive, true); -Arrays.fill(highKeyInclusive, true); -} - // determine cases when prefix search could be applied for (int i = 1; i < lowKeyExprs.length; i++) { if (lowKeyLimits[0] == null && lowKeyLimits[i] != null || lowKeyLimits[0] != null && lowKeyLimits[i] == null @@ -542,6 +537,12 @@ public class BTreeAccessMethod implements IAccessMethod { primaryIndexPostProccessingIsNeeded = true; } } + +if (primaryIndexPostProccessingIsNeeded) { +Arrays.fill(lowKeyInclusive, true); +Arrays.fill(highKeyInclusive, true); +} + if (lowKeyLimits[0] == null) { lowKeyInclusive[0] = true; } @@ -563,8 +564,10 @@ public class BTreeAccessMethod implements IAccessMethod { BTreeJobGenParams jobGenParams = new BTreeJobGenParams(chosenIndex.getIndexName(), IndexType.BTREE, dataset.getDataverseName(), dataset.getDatasetName(), retainInput, requiresBroadcast); -jobGenParams.setLowKeyInclusive(lowKeyInclusive[0]); -jobGenParams.setHighKeyInclusive(highKeyInclusive[0]); +jobGenParams + .setLowKeyInclusive(lowKeyInclusive[primaryIndexPostProccessingIsNeeded ? 0 : numSecondaryKeys - 1]); +jobGenParams + .setHighKeyInclusive(highKeyInclusive[primaryIndexPostProccessingIsNeeded ? 0 : numSecondaryKeys - 1]); jobGenParams.setIsEqCondition(isEqCondition); jobGenParams.setLowKeyVarList(keyVarList, 0, numLowKeys); jobGenParams.setHighKeyVarList(keyVarList, numLowKeys, numHighKeys); http://git-wip-us.apache.org/repos/asf/asterixdb/blob/e541f048/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/composite-key/composite-prefix/composite-prefix.1.ddl.sqlpp -- diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/composite-key/composite-prefix/composite-prefix.1.ddl.sqlpp b/as
[33/34] asterixdb git commit: [NO ISSUE] Don't break lines without whitespace by default
[NO ISSUE] Don't break lines without whitespace by default Add a new boolean indent parameter "strict" to denote when lines should be force-wrapped even when there are no word breaks. Change-Id: I716caf020466f30e469531d0bd8498d0c781c2af Reviewed-on: https://asterix-gerrit.ics.uci.edu/2574 Sonar-Qube: Jenkins Tested-by: Jenkins Contrib: Jenkins Integration-Tests: Jenkins Reviewed-by: Murtadha Hubail Project: http://git-wip-us.apache.org/repos/asf/asterixdb/repo Commit: http://git-wip-us.apache.org/repos/asf/asterixdb/commit/fa52d7c0 Tree: http://git-wip-us.apache.org/repos/asf/asterixdb/tree/fa52d7c0 Diff: http://git-wip-us.apache.org/repos/asf/asterixdb/diff/fa52d7c0 Branch: refs/heads/release-0.9.4-pre-rc Commit: fa52d7c09f4f4bd7b9aef6728efbe0235bbfe713 Parents: 3f4cb46 Author: Michael Blow Authored: Sat Apr 7 03:05:30 2018 -0400 Committer: Michael Blow Committed: Sat Apr 7 17:20:30 2018 -0700 -- .../apache/hyracks/maven/license/LicenseUtil.java | 16 ++-- .../maven/license/freemarker/IndentDirective.java | 12 +--- 2 files changed, 19 insertions(+), 9 deletions(-) -- http://git-wip-us.apache.org/repos/asf/asterixdb/blob/fa52d7c0/hyracks-fullstack/hyracks/hyracks-maven-plugins/license-automation-plugin/src/main/java/org/apache/hyracks/maven/license/LicenseUtil.java -- diff --git a/hyracks-fullstack/hyracks/hyracks-maven-plugins/license-automation-plugin/src/main/java/org/apache/hyracks/maven/license/LicenseUtil.java b/hyracks-fullstack/hyracks/hyracks-maven-plugins/license-automation-plugin/src/main/java/org/apache/hyracks/maven/license/LicenseUtil.java index a80dc1d..5ea768e 100644 --- a/hyracks-fullstack/hyracks/hyracks-maven-plugins/license-automation-plugin/src/main/java/org/apache/hyracks/maven/license/LicenseUtil.java +++ b/hyracks-fullstack/hyracks/hyracks-maven-plugins/license-automation-plugin/src/main/java/org/apache/hyracks/maven/license/LicenseUtil.java @@ -56,11 +56,11 @@ public class LicenseUtil { } } -public static String process(String input, boolean unpad, boolean wrap) throws IOException { +public static String process(String input, boolean unpad, boolean wrap, boolean strict) throws IOException { try (BufferedReader reader = new BufferedReader(new StringReader(input))) { reader.mark(input.length() + 1); StringWriter sw = new StringWriter(); -trim(sw, reader, unpad, wrap); +trim(sw, reader, unpad, wrap, strict); sw.append('\n'); return sw.toString(); } @@ -75,20 +75,22 @@ public class LicenseUtil { } private static void trim(Writer out, BufferedReader reader) throws IOException { -trim(out, reader, true, true); +trim(out, reader, true, true, false); } -private static void trim(Writer out, BufferedReader reader, boolean unpad, boolean wrap) throws IOException { +private static void trim(Writer out, BufferedReader reader, boolean unpad, boolean wrap, boolean strict) +throws IOException { Pair result = null; if (unpad || wrap) { result = analyze(reader); reader.reset(); } doTrim(out, reader, unpad ? result.getLeft() : 0, -wrap && (result.getRight() > wrapThreshold) ? wrapLength : Integer.MAX_VALUE); +wrap && (result.getRight() > wrapThreshold) ? wrapLength : Integer.MAX_VALUE, strict); } -private static void doTrim(Writer out, BufferedReader reader, int extraPadding, int wrapLength) throws IOException { +private static void doTrim(Writer out, BufferedReader reader, int extraPadding, int wrapLength, boolean strict) +throws IOException { boolean head = true; int empty = 0; for (String line = reader.readLine(); line != null; line = reader.readLine()) { @@ -110,6 +112,8 @@ public class LicenseUtil { out.append(trimmed.substring(0, cut)); out.append('\n'); trimmed = trimmed.substring(cut + 1); +} else if (!strict) { +break; } else { out.append(trimmed.substring(0, wrapLength)); out.append('\n'); http://git-wip-us.apache.org/repos/asf/asterixdb/blob/fa52d7c0/hyracks-fullstack/hyracks/hyracks-maven-plugins/license-automation-plugin/src/main/java/org/apache/hyracks/maven/license/freemarker/IndentDirective.java -- diff --git a/hyracks-fullstack/hyracks/hyracks-maven-plugins/license-automation-plugin/src/main/java/org/apache/hyracks/maven/license/freemarker/IndentDirective.ja
[18/34] asterixdb git commit: [ASTERIXDB-2345][FUN] Fix runtime output type for object_names()
[ASTERIXDB-2345][FUN] Fix runtime output type for object_names() - user model changes: no - storage format changes: no - interface changes: no Details: - object_names() should produce ordered_list instead of ordered_list Change-Id: I5181680bfe978d4208141a30b2167f78368bcf8d Reviewed-on: https://asterix-gerrit.ics.uci.edu/2540 Sonar-Qube: Jenkins Tested-by: Jenkins Contrib: Jenkins Integration-Tests: Jenkins Reviewed-by: Till Westmann Project: http://git-wip-us.apache.org/repos/asf/asterixdb/repo Commit: http://git-wip-us.apache.org/repos/asf/asterixdb/commit/6ee96a9d Tree: http://git-wip-us.apache.org/repos/asf/asterixdb/tree/6ee96a9d Diff: http://git-wip-us.apache.org/repos/asf/asterixdb/diff/6ee96a9d Branch: refs/heads/release-0.9.4-pre-rc Commit: 6ee96a9d321ea85b76077796c481d8662e3c3d0c Parents: bf41e4c Author: Dmitry Lychagin Authored: Wed Mar 28 14:36:52 2018 -0700 Committer: Dmitry Lychagin Committed: Wed Mar 28 18:25:38 2018 -0700 -- .../evaluators/functions/records/RecordNamesDescriptor.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) -- http://git-wip-us.apache.org/repos/asf/asterixdb/blob/6ee96a9d/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/records/RecordNamesDescriptor.java -- diff --git a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/records/RecordNamesDescriptor.java b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/records/RecordNamesDescriptor.java index bb8e3e7..1719980 100644 --- a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/records/RecordNamesDescriptor.java +++ b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/records/RecordNamesDescriptor.java @@ -31,6 +31,7 @@ import org.apache.asterix.om.pointables.nonvisitor.ARecordPointable; import org.apache.asterix.om.types.AOrderedListType; import org.apache.asterix.om.types.ARecordType; import org.apache.asterix.om.types.ATypeTag; +import org.apache.asterix.om.types.BuiltinType; import org.apache.asterix.runtime.evaluators.base.AbstractScalarFunctionDynamicDescriptor; import org.apache.asterix.runtime.evaluators.functions.PointableHelper; import org.apache.asterix.runtime.functions.FunctionTypeInferers; @@ -78,6 +79,7 @@ public class RecordNamesDescriptor extends AbstractScalarFunctionDynamicDescript private final IPointable argPtr = new VoidPointable(); private final ARecordPointable recordPointable = (ARecordPointable) ARecordPointable.FACTORY.createPointable(); +private final AOrderedListType listType = new AOrderedListType(BuiltinType.ASTRING, null); private final OrderedListBuilder listBuilder = new OrderedListBuilder(); private final ArrayBackedValueStorage itemStorage = new ArrayBackedValueStorage(); private final DataOutput itemOut = itemStorage.getDataOutput(); @@ -100,7 +102,7 @@ public class RecordNamesDescriptor extends AbstractScalarFunctionDynamicDescript recordPointable.set(data, offset, argPtr.getLength()); - listBuilder.reset(AOrderedListType.FULL_OPEN_ORDEREDLIST_TYPE); +listBuilder.reset(listType); try { for (int i = 0, n = recordPointable.getSchemeFieldCount(recType); i < n; i++) {
[21/34] asterixdb git commit: [ASTERIXDB-2346][COMP] Constant folding should not fail on runtime exceptions
[ASTERIXDB-2346][COMP] Constant folding should not fail on runtime exceptions - user model changes: no - storage format changes: no - interface changes: no Details: - Constant folding rule should not fail on runtime exceptions - throw RuntimeDataException instead of java.lang.ArithmeticException from numeric operators Change-Id: I286551a98f57df798ce982228a66d6a1e3fc7304 Reviewed-on: https://asterix-gerrit.ics.uci.edu/2542 Sonar-Qube: Jenkins Integration-Tests: Jenkins Tested-by: Jenkins Contrib: Jenkins Reviewed-by: Till Westmann Project: http://git-wip-us.apache.org/repos/asf/asterixdb/repo Commit: http://git-wip-us.apache.org/repos/asf/asterixdb/commit/cc9257eb Tree: http://git-wip-us.apache.org/repos/asf/asterixdb/tree/cc9257eb Diff: http://git-wip-us.apache.org/repos/asf/asterixdb/diff/cc9257eb Branch: refs/heads/release-0.9.4-pre-rc Commit: cc9257ebffe118bd85d76ae915ad03fe62503eb9 Parents: 8e4b57b Author: Dmitry Lychagin Authored: Wed Mar 28 14:52:57 2018 -0700 Committer: Dmitry Lychagin Committed: Thu Mar 29 22:22:41 2018 -0700 -- .../optimizer/rules/ConstantFoldingRule.java| 71 +++- .../numeric/ifinf/ifinf.1.query.sqlpp | 3 +- .../runtimets/results/numeric/ifinf/ifinf.1.adm | 3 +- .../asterix/common/exceptions/ErrorCode.java| 1 + .../main/resources/asx_errormsg/en.properties | 1 + .../functions/NumericCaretDescriptor.java | 3 +- .../functions/NumericDivideDescriptor.java | 7 +- 7 files changed, 52 insertions(+), 37 deletions(-) -- http://git-wip-us.apache.org/repos/asf/asterixdb/blob/cc9257eb/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/ConstantFoldingRule.java -- diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/ConstantFoldingRule.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/ConstantFoldingRule.java index fd66821..7e9328b 100644 --- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/ConstantFoldingRule.java +++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/ConstantFoldingRule.java @@ -67,6 +67,7 @@ import org.apache.hyracks.algebricks.core.algebra.functions.FunctionIdentifier; import org.apache.hyracks.algebricks.core.algebra.operators.logical.IOperatorSchema; import org.apache.hyracks.algebricks.core.algebra.visitors.ILogicalExpressionReferenceTransform; import org.apache.hyracks.algebricks.core.algebra.visitors.ILogicalExpressionVisitor; +import org.apache.hyracks.algebricks.core.config.AlgebricksConfig; import org.apache.hyracks.algebricks.core.jobgen.impl.JobGenContext; import org.apache.hyracks.algebricks.core.rewriter.base.IAlgebraicRewriteRule; import org.apache.hyracks.algebricks.runtime.base.IScalarEvaluator; @@ -93,11 +94,13 @@ public class ConstantFoldingRule implements IAlgebraicRewriteRule { BuiltinFunctions.FIELD_ACCESS_BY_INDEX, BuiltinFunctions.CAST_TYPE, BuiltinFunctions.META, BuiltinFunctions.META_KEY, BuiltinFunctions.RECORD_CONCAT, BuiltinFunctions.RECORD_CONCAT_STRICT); -/** Throws exceptions in substituiteProducedVariable, setVarType, and one getVarType method. */ +/** + * Throws exceptions in substituiteProducedVariable, setVarType, and one getVarType method. + */ private static final IVariableTypeEnvironment _emptyTypeEnv = new IVariableTypeEnvironment() { @Override -public boolean substituteProducedVariable(LogicalVariable v1, LogicalVariable v2) throws AlgebricksException { +public boolean substituteProducedVariable(LogicalVariable v1, LogicalVariable v2) { throw new IllegalStateException(); } @@ -108,12 +111,12 @@ public class ConstantFoldingRule implements IAlgebraicRewriteRule { @Override public Object getVarType(LogicalVariable var, List nonNullVariables, -List> correlatedNullableVariableLists) throws AlgebricksException { +List> correlatedNullableVariableLists) { throw new IllegalStateException(); } @Override -public Object getVarType(LogicalVariable var) throws AlgebricksException { +public Object getVarType(LogicalVariable var) { throw new IllegalStateException(); } @@ -170,14 +173,13 @@ public class ConstantFoldingRule implements IAlgebraicRewriteRule { } @Override -public Pair visitConstantExpression(ConstantExpression expr, Void arg) -throws AlgebricksException { +public Pair visitConstantExpression(ConstantExpression expr, Void arg) { return new Pair<>(false, expr); } @Override publ
[04/34] asterixdb git commit: [NO ISSUE][CLUS] Exclude Pending Removal Nodes From Location
[NO ISSUE][CLUS] Exclude Pending Removal Nodes From Location - user model changes: no - storage format changes: no - interface changes: no Details: - Exclude pending removel nodes from cluster locations to prevent new jobs from using them as compute locations. - Add additional logging for dataset rebalance. Change-Id: I0c9904b229aa086615a3a8bbdfaf44d51f1e1757 Reviewed-on: https://asterix-gerrit.ics.uci.edu/2523 Reviewed-by: Michael Blow Integration-Tests: Michael Blow Tested-by: Michael Blow Project: http://git-wip-us.apache.org/repos/asf/asterixdb/repo Commit: http://git-wip-us.apache.org/repos/asf/asterixdb/commit/ad1d30ab Tree: http://git-wip-us.apache.org/repos/asf/asterixdb/tree/ad1d30ab Diff: http://git-wip-us.apache.org/repos/asf/asterixdb/diff/ad1d30ab Branch: refs/heads/release-0.9.4-pre-rc Commit: ad1d30abb32174bc9d7ada81bd8239036e171df2 Parents: 94b6da6 Author: Murtadha Hubail Authored: Sun Mar 25 04:59:34 2018 +0300 Committer: Murtadha Hubail Committed: Sun Mar 25 07:34:58 2018 -0700 -- .../src/main/java/org/apache/asterix/utils/RebalanceUtil.java | 7 +++ .../org/apache/asterix/runtime/utils/ClusterStateManager.java | 1 + 2 files changed, 8 insertions(+) -- http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ad1d30ab/asterixdb/asterix-app/src/main/java/org/apache/asterix/utils/RebalanceUtil.java -- diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/utils/RebalanceUtil.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/utils/RebalanceUtil.java index 7bb917f..702b6b4 100644 --- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/utils/RebalanceUtil.java +++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/utils/RebalanceUtil.java @@ -124,6 +124,9 @@ public class RebalanceUtil { // The target dataset for rebalance. targetDataset = sourceDataset.getTargetDatasetForRebalance(nodeGroupName); +LOGGER.info("Rebalancing dataset {} from node group {} with nodes {} to node group {} with nodes {}", +sourceDataset.getDatasetName(), sourceDataset.getNodeGroupName(), sourceNodes, +targetDataset.getNodeGroupName(), targetNcNames); // Rebalances the source dataset into the target dataset. rebalance(sourceDataset, targetDataset, metadataProvider, hcc, datasetRebalanceCallback); } else { @@ -158,6 +161,7 @@ public class RebalanceUtil { // the source dataset. runMetadataTransaction(metadataProvider, () -> dropSourceDataset(sourceDataset, metadataProvider, hcc)); }); +LOGGER.info("Dataset {} rebalance completed successfully", datasetName); } @FunctionalInterface @@ -238,6 +242,8 @@ public class RebalanceUtil { (ActiveNotificationHandler) appCtx.getActiveNotificationHandler(); IMetadataLockManager lockManager = appCtx.getMetadataLockManager(); lockManager.upgradeDatasetLockToWrite(metadataProvider.getLocks(), DatasetUtil.getFullyQualifiedName(source)); +LOGGER.info("Updating dataset {} node group from {} to {}", source.getDatasetName(), source.getNodeGroupName(), +target.getNodeGroupName()); try { // Updates the dataset entry in the metadata storage MetadataManager.INSTANCE.updateDataset(mdTxnCtx, target); @@ -248,6 +254,7 @@ public class RebalanceUtil { } } MetadataManager.INSTANCE.commitTransaction(mdTxnCtx); +LOGGER.info("dataset {} node group updated to {}", target.getDatasetName(), target.getNodeGroupName()); } finally { lockManager.downgradeDatasetLockToExclusiveModify(metadataProvider.getLocks(), DatasetUtil.getFullyQualifiedName(target)); http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ad1d30ab/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/utils/ClusterStateManager.java -- diff --git a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/utils/ClusterStateManager.java b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/utils/ClusterStateManager.java index 7182204..73d6705 100644 --- a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/utils/ClusterStateManager.java +++ b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/utils/ClusterStateManager.java @@ -280,6 +280,7 @@ public class ClusterStateManager implements IClusterStateManager { clusterActiveLocations.add(p.getActiveNodeId()); } } +clusterActiveLocations.removeAll(pendingRemo
[08/34] asterixdb git commit: [NO ISSUE] Cleanup query service exception handling
[NO ISSUE] Cleanup query service exception handling Change-Id: I1f2828481df055d6c96f1ae1869ef37a065bf576 Reviewed-on: https://asterix-gerrit.ics.uci.edu/2524 Sonar-Qube: Jenkins Tested-by: Jenkins Contrib: Jenkins Integration-Tests: Jenkins Reviewed-by: Murtadha Hubail Project: http://git-wip-us.apache.org/repos/asf/asterixdb/repo Commit: http://git-wip-us.apache.org/repos/asf/asterixdb/commit/c4eb7b14 Tree: http://git-wip-us.apache.org/repos/asf/asterixdb/tree/c4eb7b14 Diff: http://git-wip-us.apache.org/repos/asf/asterixdb/diff/c4eb7b14 Branch: refs/heads/release-0.9.4-pre-rc Commit: c4eb7b143a09edee73e3fa5c91b94dcda5413eaa Parents: e541f04 Author: Michael Blow Authored: Mon Mar 26 20:07:32 2018 -0400 Committer: Michael Blow Committed: Mon Mar 26 20:21:44 2018 -0700 -- .../api/http/server/NCQueryServiceServlet.java | 8 ++-- .../api/http/server/QueryServiceServlet.java| 48 ++-- .../message/ExecuteStatementRequestMessage.java | 14 +++--- .../asterix/common/exceptions/ErrorCode.java| 2 + .../main/resources/asx_errormsg/en.properties | 2 + .../common/exceptions/AlgebricksException.java | 4 ++ .../api/exceptions/HyracksException.java| 4 +- 7 files changed, 54 insertions(+), 28 deletions(-) -- http://git-wip-us.apache.org/repos/asf/asterixdb/blob/c4eb7b14/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/NCQueryServiceServlet.java -- diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/NCQueryServiceServlet.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/NCQueryServiceServlet.java index 3564736..a420efc 100644 --- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/NCQueryServiceServlet.java +++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/NCQueryServiceServlet.java @@ -154,13 +154,13 @@ public class NCQueryServiceServlet extends QueryServiceServlet { } @Override -protected void handleExecuteStatementException(Throwable t, RequestExecutionState execution) { -if (t instanceof TimeoutException +protected void handleExecuteStatementException(Throwable t, RequestExecutionState state, RequestParameters param) { +if (t instanceof TimeoutException // TODO(mblow): I don't think t can ever been an instance of TimeoutException || ExceptionUtils.matchingCause(t, candidate -> candidate instanceof IPCException)) { GlobalConfig.ASTERIX_LOGGER.log(Level.WARN, t.toString(), t); -execution.setStatus(ResultStatus.FAILED, HttpResponseStatus.SERVICE_UNAVAILABLE); +state.setStatus(ResultStatus.FAILED, HttpResponseStatus.SERVICE_UNAVAILABLE); } else { -super.handleExecuteStatementException(t, execution); +super.handleExecuteStatementException(t, state, param); } } http://git-wip-us.apache.org/repos/asf/asterixdb/blob/c4eb7b14/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/QueryServiceServlet.java -- diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/QueryServiceServlet.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/QueryServiceServlet.java index 1057a73..56359e3 100644 --- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/QueryServiceServlet.java +++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/QueryServiceServlet.java @@ -18,6 +18,11 @@ */ package org.apache.asterix.api.http.server; +import static org.apache.asterix.common.exceptions.ErrorCode.ASTERIX; +import static org.apache.asterix.common.exceptions.ErrorCode.QUERY_TIMEOUT; +import static org.apache.asterix.common.exceptions.ErrorCode.REJECT_BAD_CLUSTER_STATE; +import static org.apache.asterix.common.exceptions.ErrorCode.REJECT_NODE_UNREGISTERED; + import java.io.IOException; import java.io.PrintWriter; import java.net.InetAddress; @@ -35,7 +40,6 @@ import org.apache.asterix.common.config.GlobalConfig; import org.apache.asterix.common.context.IStorageComponentProvider; import org.apache.asterix.common.dataflow.ICcApplicationContext; import org.apache.asterix.common.exceptions.AsterixException; -import org.apache.asterix.common.exceptions.ErrorCode; import org.apache.asterix.compiler.provider.ILangCompilationProvider; import org.apache.asterix.lang.aql.parser.TokenMgrError; import org.apache.asterix.lang.common.base.IParser; @@ -69,7 +73,6 @@ import com.fasterxml.jackson.databind.JsonMappingException; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; import com.fa
[03/34] asterixdb git commit: [NO ISSUE] IOCounter improvements
[NO ISSUE] IOCounter improvements - cache results for short period - split iostat, proc fs impls Change-Id: I7789171db6b6d7eea3561c24467af63f065f5dc6 Reviewed-on: https://asterix-gerrit.ics.uci.edu/2515 Reviewed-by: Murtadha Hubail Tested-by: Murtadha Hubail Sonar-Qube: Jenkins Contrib: Jenkins Tested-by: Jenkins Integration-Tests: Jenkins Project: http://git-wip-us.apache.org/repos/asf/asterixdb/repo Commit: http://git-wip-us.apache.org/repos/asf/asterixdb/commit/94b6da6a Tree: http://git-wip-us.apache.org/repos/asf/asterixdb/tree/94b6da6a Diff: http://git-wip-us.apache.org/repos/asf/asterixdb/diff/94b6da6a Branch: refs/heads/release-0.9.4-pre-rc Commit: 94b6da6ae33749dfdee35e35f32c165693015e3d Parents: 2587f1d Author: Michael Blow Authored: Thu Mar 22 20:15:37 2018 -0400 Committer: Michael Blow Committed: Thu Mar 22 22:11:31 2018 -0700 -- .../control/nc/io/profiling/IIOCounter.java | 8 +- .../control/nc/io/profiling/IOCounterCache.java | 41 ++ .../nc/io/profiling/IOCounterDefault.java | 6 +- .../nc/io/profiling/IOCounterFactory.java | 9 +- .../nc/io/profiling/IOCounterIoStat.java| 101 +++ .../control/nc/io/profiling/IOCounterLinux.java | 128 --- .../control/nc/io/profiling/IOCounterOSX.java | 6 +- .../control/nc/io/profiling/IOCounterProc.java | 71 ++ 8 files changed, 231 insertions(+), 139 deletions(-) -- http://git-wip-us.apache.org/repos/asf/asterixdb/blob/94b6da6a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/io/profiling/IIOCounter.java -- diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/io/profiling/IIOCounter.java b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/io/profiling/IIOCounter.java index 3612d8f..a85ca2c 100644 --- a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/io/profiling/IIOCounter.java +++ b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/io/profiling/IIOCounter.java @@ -22,12 +22,12 @@ package org.apache.hyracks.control.nc.io.profiling; public interface IIOCounter { /** - * @return the number of block reads from the very beginning + * @return the number of block reads from the very beginning, or -1 if not available on this platform */ -public long getReads(); +long getReads(); /** - * @return the number of block writes from the very beginning + * @return the number of block writes from the very beginning, or -1 if not available on this platform */ -public long getWrites(); +long getWrites(); } http://git-wip-us.apache.org/repos/asf/asterixdb/blob/94b6da6a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/io/profiling/IOCounterCache.java -- diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/io/profiling/IOCounterCache.java b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/io/profiling/IOCounterCache.java new file mode 100644 index 000..842d82b --- /dev/null +++ b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/io/profiling/IOCounterCache.java @@ -0,0 +1,41 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hyracks.control.nc.io.profiling; + +import java.io.IOException; +import java.util.concurrent.TimeUnit; + +import org.apache.hyracks.util.Span; + +abstract class IOCounterCache implements IIOCounter { +private static final long TTL_NANOS = TimeUnit.MILLISECONDS.toNanos(500); +private Span span; +private T info; + +
[25/34] asterixdb git commit: [NO ISSUE][STO] Misc fixes in storage
[NO ISSUE][STO] Misc fixes in storage - user model changes: no - storage format changes: no - interface changes: no Details: - Blocking IO callback used to wait for any notification on the callback before returning. The behaviour was fixed to only return if the completion flag was set on afterFinalize. - Reading and writing to and from memory component's didn't do any locking and so, this could read to concurrency issues. - Reading metadata values used to rely on pointables which can be problematic because then the caller will need to latch/pin the page correctly. To avoid this, readers of metadata pages will always take a copy of the metadata. Change-Id: I4bdc4c16a9c126d311378e56651632bbb4a50864 Reviewed-on: https://asterix-gerrit.ics.uci.edu/2548 Sonar-Qube: Jenkins Reviewed-by: abdullah alamoudi Tested-by: Jenkins Contrib: Jenkins Integration-Tests: Jenkins Reviewed-by: Murtadha Hubail Project: http://git-wip-us.apache.org/repos/asf/asterixdb/repo Commit: http://git-wip-us.apache.org/repos/asf/asterixdb/commit/b1843b24 Tree: http://git-wip-us.apache.org/repos/asf/asterixdb/tree/b1843b24 Diff: http://git-wip-us.apache.org/repos/asf/asterixdb/diff/b1843b24 Branch: refs/heads/release-0.9.4-pre-rc Commit: b1843b244be4a4e542ecb7e1e0845bd1d2c7cb71 Parents: ee8526b Author: Abdullah Alamoudi Authored: Fri Mar 30 15:12:15 2018 -0700 Committer: abdullah alamoudi Committed: Fri Mar 30 17:13:02 2018 -0700 -- .../test/dataflow/FlushMetadataOnlyTest.java| 4 +- .../asterix/test/dataflow/LogMarkerTest.java| 8 +- .../AbstractLSMIOOperationCallback.java | 4 +- .../PrimaryIndexLogMarkerCallback.java | 14 +++- .../src/main/resources/errormsg/en.properties | 2 +- .../data/std/util/ArrayBackedValueStorage.java | 48 +-- .../hyracks/data/std/util/GrowableArray.java| 20 - .../am/lsm/common/api/IComponentMetadata.java | 13 +-- .../common/impls/AbstractLSMDiskComponent.java | 4 +- .../BlockingIOOperationCallbackWrapper.java | 2 +- .../lsm/common/impls/DiskComponentMetadata.java | 12 +-- .../impls/EmptyDiskComponentMetadata.java | 9 +- .../storage/am/lsm/common/impls/LSMHarness.java | 11 ++- .../common/impls/MemoryComponentMetadata.java | 87 ++-- .../am/lsm/common/util/ComponentUtils.java | 43 ++ .../am/lsm/common/util/LSMComponentIdUtils.java | 8 +- 16 files changed, 187 insertions(+), 102 deletions(-) -- http://git-wip-us.apache.org/repos/asf/asterixdb/blob/b1843b24/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/dataflow/FlushMetadataOnlyTest.java -- diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/dataflow/FlushMetadataOnlyTest.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/dataflow/FlushMetadataOnlyTest.java index f9421a1..1251d91 100644 --- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/dataflow/FlushMetadataOnlyTest.java +++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/dataflow/FlushMetadataOnlyTest.java @@ -28,7 +28,7 @@ import org.apache.asterix.test.base.TestMethodTracer; import org.apache.asterix.test.common.TestHelper; import org.apache.hyracks.api.context.IHyracksTaskContext; import org.apache.hyracks.api.job.JobId; -import org.apache.hyracks.data.std.primitive.VoidPointable; +import org.apache.hyracks.data.std.util.ArrayBackedValueStorage; import org.apache.hyracks.data.std.util.DataUtils; import org.apache.hyracks.storage.am.common.api.IIndexDataflowHelper; import org.apache.hyracks.storage.am.common.dataflow.IndexDataflowHelperFactory; @@ -106,7 +106,7 @@ public class FlushMetadataOnlyTest { StorageTestUtils.flush(dsLifecycleMgr, lsmBtree, false); // assert one disk component Assert.assertEquals(1, lsmBtree.getDiskComponents().size()); -VoidPointable pointable = VoidPointable.FACTORY.createPointable(); +ArrayBackedValueStorage pointable = new ArrayBackedValueStorage(); ComponentUtils.get(lsmBtree, key, pointable); Assert.assertTrue(DataUtils.equals(pointable, value)); // ensure that we can search this component http://git-wip-us.apache.org/repos/asf/asterixdb/blob/b1843b24/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/dataflow/LogMarkerTest.java -- diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/dataflow/LogMarkerTest.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/dataflow/LogMarkerTest.java index 0a968c8..2121327 100644 --- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/dataflow/LogMarkerTest.java +++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/
[14/34] asterixdb git commit: [NO ISSUE] Incremental cleanup of deprecated exception ctors
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/6453ecf2/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/StringRepeatDescriptor.java -- diff --git a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/StringRepeatDescriptor.java b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/StringRepeatDescriptor.java index 77c4f20..4672c75 100644 --- a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/StringRepeatDescriptor.java +++ b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/StringRepeatDescriptor.java @@ -114,7 +114,7 @@ public class StringRepeatDescriptor extends AbstractScalarFunctionDynamicDescrip out.write(bytes, inputStringStart, inputLen); } } catch (IOException e) { -throw new HyracksDataException(e); +throw HyracksDataException.create(e); } result.set(resultStorage); } http://git-wip-us.apache.org/repos/asf/asterixdb/blob/6453ecf2/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/StringSplitDescriptor.java -- diff --git a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/StringSplitDescriptor.java b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/StringSplitDescriptor.java index f788366..4cfb92d 100644 --- a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/StringSplitDescriptor.java +++ b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/StringSplitDescriptor.java @@ -141,7 +141,7 @@ public class StringSplitDescriptor extends AbstractScalarFunctionDynamicDescript listBuilder.write(out, true); result.set(resultStorage); } catch (IOException e1) { -throw new HyracksDataException(e1); +throw HyracksDataException.create(e1); } } http://git-wip-us.apache.org/repos/asf/asterixdb/blob/6453ecf2/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/StringToCodePointDescriptor.java -- diff --git a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/StringToCodePointDescriptor.java b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/StringToCodePointDescriptor.java index fe5ad9f..63d474d 100644 --- a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/StringToCodePointDescriptor.java +++ b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/StringToCodePointDescriptor.java @@ -108,7 +108,7 @@ public class StringToCodePointDescriptor extends AbstractScalarFunctionDynamicDe } result.set(resultStorage); } catch (IOException e1) { -throw new HyracksDataException(e1); +throw HyracksDataException.create(e1); } } }; http://git-wip-us.apache.org/repos/asf/asterixdb/blob/6453ecf2/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/Substring2Descriptor.java -- diff --git a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/Substring2Descriptor.java b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/Substring2Descriptor.java index 8ebfb94..53f73ae 100644 --- a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/Substring2Descriptor.java +++ b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/Substring2Descriptor.java @@ -102,13 +102,13 @@ public class Substring2Descriptor extends AbstractStringOffsetConfigurableDescri } catch (StringIndexOutOfBoundsException e) { throw new RuntimeDataException(ErrorCode.OUT_OF_BOUND, getIdentifier(), 1, start); } catch (IOException e) { -throw new HyracksDataException(e); +throw HyracksDataException.create(e);
[16/34] asterixdb git commit: [NO ISSUE] Incremental cleanup of deprecated exception ctors
[NO ISSUE] Incremental cleanup of deprecated exception ctors Change-Id: I1e7c3655828fc6530cef83ea502a6cfbf41acddf Reviewed-on: https://asterix-gerrit.ics.uci.edu/2533 Integration-Tests: Jenkins Tested-by: Jenkins Contrib: Jenkins Reviewed-by: Murtadha Hubail Project: http://git-wip-us.apache.org/repos/asf/asterixdb/repo Commit: http://git-wip-us.apache.org/repos/asf/asterixdb/commit/6453ecf2 Tree: http://git-wip-us.apache.org/repos/asf/asterixdb/tree/6453ecf2 Diff: http://git-wip-us.apache.org/repos/asf/asterixdb/diff/6453ecf2 Branch: refs/heads/release-0.9.4-pre-rc Commit: 6453ecf2f0dff1ec1306c8c5c1e4f81a27755431 Parents: f3784bb Author: Michael Blow Authored: Tue Mar 27 00:00:28 2018 -0400 Committer: Michael Blow Committed: Tue Mar 27 14:32:35 2018 -0700 -- .../asterix/app/result/ResultPrinter.java | 4 ++-- .../asterix/app/translator/QueryTranslator.java | 2 +- .../src/test/resources/runtimets/testsuite.xml | 4 ++-- .../common/config/AsterixProperties.java| 2 +- .../context/PrimaryIndexOperationTracker.java | 2 +- .../LSMInsertDeleteOperatorNodePushable.java| 2 +- .../dataflow/FeedStreamDataFlowController.java | 2 +- .../dataflow/RecordDataFlowController.java | 2 +- .../dataflow/StreamDataFlowController.java | 2 +- .../external/dataset/adapter/LookupAdapter.java | 4 ++-- .../external/feed/dataflow/FrameSpiller.java| 6 +++--- .../external/indexing/FileOffsetIndexer.java| 4 ++-- .../external/indexing/IndexingScheduler.java| 6 +++--- .../indexing/RecordColumnarIndexer.java | 4 ++-- .../external/input/HDFSDataSourceFactory.java | 2 +- .../record/reader/hdfs/HDFSRecordReader.java| 2 +- .../reader/rss/RSSRecordReaderFactory.java | 2 +- .../stream/StreamRecordReaderFactory.java | 4 +--- .../reader/twitter/TwitterPullRecordReader.java | 2 +- .../twitter/TwitterRecordReaderFactory.java | 2 +- .../external/input/stream/HDFSInputStream.java | 2 +- .../library/ExternalFunctionProvider.java | 6 +++--- .../external/library/java/base/JObject.java | 2 +- .../external/library/java/base/JRecord.java | 2 +- ...xternalDatasetIndexesOperatorDescriptor.java | 2 +- ...ExternalBTreeSearchOperatorNodePushable.java | 2 +- ...rnalIndexBulkModifyOperatorNodePushable.java | 2 +- .../ExternalLookupOperatorDescriptor.java | 6 +++--- ...ExternalRTreeSearchOperatorNodePushable.java | 4 ++-- .../ExternalScanOperatorDescriptor.java | 2 +- .../FeedCollectOperatorNodePushable.java| 2 +- .../operators/FeedMetaComputeNodePushable.java | 4 ++-- .../operators/FeedMetaStoreNodePushable.java| 2 +- .../external/parser/DelimitedDataParser.java| 8 +++ .../external/parser/HiveRecordParser.java | 6 +++--- .../parser/RecordWithMetadataParser.java| 4 ++-- .../external/parser/RecordWithPKDataParser.java | 2 +- .../asterix/external/parser/TweetParser.java| 2 +- .../provider/DataflowControllerProvider.java| 2 +- .../asterix/external/util/FeedLogManager.java | 2 +- .../external/util/FileSystemWatcher.java| 2 +- .../asterix/external/classad/ClassAd.java | 2 +- .../asterix/external/library/ClassAdParser.java | 4 ++-- .../common/EditDistanceCheckEvaluator.java | 2 +- .../common/EditDistanceEvaluator.java | 2 +- .../evaluators/common/GramTokensEvaluator.java | 2 +- .../common/SimilarityJaccardCheckEvaluator.java | 2 +- .../common/SimilarityJaccardEvaluator.java | 2 +- .../SimilarityJaccardPrefixEvaluator.java | 2 +- .../evaluators/common/WordTokensEvaluator.java | 2 +- .../EditDistanceListIsFilterableDescriptor.java | 2 +- ...EditDistanceStringIsFilterableEvaluator.java | 2 +- .../functions/PrefixLenDescriptor.java | 2 +- .../functions/PrefixLenJaccardDescriptor.java | 2 +- .../DatatypeTupleTranslator.java| 2 +- .../asterix/builders/AbstractListBuilder.java | 2 +- .../apache/asterix/builders/RecordBuilder.java | 2 +- .../data/nontagged/MissingWriterFactory.java| 2 +- .../hash/ListItemBinaryHashFunctionFactory.java | 2 +- .../data/nontagged/printers/PrintTools.java | 12 +-- .../printers/adm/ABinaryHexPrinterFactory.java | 2 +- .../printers/adm/AInt16PrinterFactory.java | 2 +- .../printers/adm/AInt32PrinterFactory.java | 2 +- .../printers/adm/AInt64PrinterFactory.java | 2 +- .../printers/adm/AInt8PrinterFactory.java | 2 +- .../printers/adm/AStringPrinterFactory.java | 2 +- .../printers/csv/ABinaryHexPrinterFactory.java | 2 +- .../printers/csv/AStringPrinterFactory.java | 2 +- .../json/clean/ABinaryHexPrinterFactory.java| 2 +- .../json/clean/AStringPrinterFactory.java | 2 +- .../clean/AUnorderedlistPrinterFactory.java | 2 +- .../json/lossless/ABinaryHexPrinterFactory.java
[30/34] asterixdb git commit: [NO ISSUE] Support dir names in generated license files
[NO ISSUE] Support dir names in generated license files Change-Id: I5af9d85fc95835d6177bdd32c9df6306e87255e3 Reviewed-on: https://asterix-gerrit.ics.uci.edu/2565 Sonar-Qube: Jenkins Reviewed-by: Murtadha Hubail Tested-by: Jenkins Contrib: Jenkins Project: http://git-wip-us.apache.org/repos/asf/asterixdb/repo Commit: http://git-wip-us.apache.org/repos/asf/asterixdb/commit/da479dc4 Tree: http://git-wip-us.apache.org/repos/asf/asterixdb/tree/da479dc4 Diff: http://git-wip-us.apache.org/repos/asf/asterixdb/diff/da479dc4 Branch: refs/heads/release-0.9.4-pre-rc Commit: da479dc466dfbeae49b154b2b59a40a27cdd19aa Parents: aa05501 Author: Michael Blow Authored: Tue Apr 3 23:05:05 2018 -0400 Committer: Michael Blow Committed: Tue Apr 3 21:12:58 2018 -0700 -- .../org/apache/hyracks/maven/license/GenerateFileMojo.java | 2 +- .../org/apache/hyracks/test/support/LicensingTestBase.java | 8 ++-- 2 files changed, 7 insertions(+), 3 deletions(-) -- http://git-wip-us.apache.org/repos/asf/asterixdb/blob/da479dc4/hyracks-fullstack/hyracks/hyracks-maven-plugins/license-automation-plugin/src/main/java/org/apache/hyracks/maven/license/GenerateFileMojo.java -- diff --git a/hyracks-fullstack/hyracks/hyracks-maven-plugins/license-automation-plugin/src/main/java/org/apache/hyracks/maven/license/GenerateFileMojo.java b/hyracks-fullstack/hyracks/hyracks-maven-plugins/license-automation-plugin/src/main/java/org/apache/hyracks/maven/license/GenerateFileMojo.java index 0245eb3..e8625fc 100644 --- a/hyracks-fullstack/hyracks/hyracks-maven-plugins/license-automation-plugin/src/main/java/org/apache/hyracks/maven/license/GenerateFileMojo.java +++ b/hyracks-fullstack/hyracks/hyracks-maven-plugins/license-automation-plugin/src/main/java/org/apache/hyracks/maven/license/GenerateFileMojo.java @@ -182,8 +182,8 @@ public class GenerateFileMojo extends LicenseMojo { throw new IOException("Could not load template " + generation.getTemplate()); } -outputDir.mkdirs(); final File file = new File(outputDir, generation.getOutputFile()); +file.getParentFile().mkdirs(); getLog().info("Writing " + file + "..."); try (final FileOutputStream fos = new FileOutputStream(file); final Writer writer = new OutputStreamWriter(fos, StandardCharsets.UTF_8)) { http://git-wip-us.apache.org/repos/asf/asterixdb/blob/da479dc4/hyracks-fullstack/hyracks/hyracks-test-support/src/main/java/org/apache/hyracks/test/support/LicensingTestBase.java -- diff --git a/hyracks-fullstack/hyracks/hyracks-test-support/src/main/java/org/apache/hyracks/test/support/LicensingTestBase.java b/hyracks-fullstack/hyracks/hyracks-test-support/src/main/java/org/apache/hyracks/test/support/LicensingTestBase.java index 2fb8446..e2ab3a3 100644 --- a/hyracks-fullstack/hyracks/hyracks-test-support/src/main/java/org/apache/hyracks/test/support/LicensingTestBase.java +++ b/hyracks-fullstack/hyracks/hyracks-test-support/src/main/java/org/apache/hyracks/test/support/LicensingTestBase.java @@ -55,7 +55,7 @@ public abstract class LicensingTestBase { protected void verifyMissingLicenses() throws IOException { for (String licenseArtifactName : getLicenseArtifactNames()) { final File licenseFile = -new File(FileUtil.joinPath(installerDir, pathToLicensingFiles(), licenseArtifactName)); +new File(FileUtil.joinPath(getInstallerDir(), pathToLicensingFiles(), licenseArtifactName)); List badLines = new ArrayList<>(); for (String line : FileUtils.readLines(licenseFile, StandardCharsets.UTF_8)) { if (line.matches("^\\s*MISSING:.*")) { @@ -66,9 +66,13 @@ public abstract class LicensingTestBase { } } +protected String getInstallerDir() { +return installerDir; +} + protected void verifyAllRequiredArtifactsPresent() { for (String name : getRequiredArtifactNames()) { -final String fileName = FileUtil.joinPath(installerDir, pathToLicensingFiles(), name); +final String fileName = FileUtil.joinPath(getInstallerDir(), pathToLicensingFiles(), name); Assert.assertTrue(fileName + " missing", new File(fileName).exists()); } }
[29/34] asterixdb git commit: [ASTERIXDB-2353][HYR][RT][FAIL] Provide complete thread dumps
[ASTERIXDB-2353][HYR][RT][FAIL] Provide complete thread dumps - user model changes: no - storage format changes: no - interface changes: no Details: - Implements a thread dump mechanism that does not truncate stack frames after the top 8 Change-Id: Id778615b3ac8951113d6b9ea027ad8650b784cb2 Reviewed-on: https://asterix-gerrit.ics.uci.edu/2564 Sonar-Qube: Jenkins Reviewed-by: Murtadha Hubail Tested-by: Jenkins Contrib: Jenkins Project: http://git-wip-us.apache.org/repos/asf/asterixdb/repo Commit: http://git-wip-us.apache.org/repos/asf/asterixdb/commit/aa05501d Tree: http://git-wip-us.apache.org/repos/asf/asterixdb/tree/aa05501d Diff: http://git-wip-us.apache.org/repos/asf/asterixdb/diff/aa05501d Branch: refs/heads/release-0.9.4-pre-rc Commit: aa05501d4cd23eae0210e9da7e4b22721290debc Parents: 5ff35c9 Author: Michael Blow Authored: Tue Apr 3 18:49:03 2018 -0400 Committer: Michael Blow Committed: Tue Apr 3 16:55:24 2018 -0700 -- .../org/apache/hyracks/util/ThreadDumpUtil.java | 64 +++- 1 file changed, 61 insertions(+), 3 deletions(-) -- http://git-wip-us.apache.org/repos/asf/asterixdb/blob/aa05501d/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/ThreadDumpUtil.java -- diff --git a/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/ThreadDumpUtil.java b/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/ThreadDumpUtil.java index 221d4b0..2de6700 100644 --- a/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/ThreadDumpUtil.java +++ b/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/ThreadDumpUtil.java @@ -19,7 +19,9 @@ package org.apache.hyracks.util; import java.io.IOException; +import java.lang.management.LockInfo; import java.lang.management.ManagementFactory; +import java.lang.management.MonitorInfo; import java.lang.management.ThreadInfo; import java.lang.management.ThreadMXBean; import java.util.ArrayList; @@ -29,6 +31,8 @@ import java.util.List; import java.util.Map; import java.util.stream.Stream; +import org.apache.commons.lang3.mutable.MutableInt; + import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.SerializationFeature; import com.fasterxml.jackson.databind.node.ObjectNode; @@ -100,8 +104,62 @@ public class ThreadDumpUtil { } public static String takeDumpString() { -StringBuilder buf = new StringBuilder(2048); -Stream.of(threadMXBean.dumpAllThreads(true, true)).forEach(buf::append); -return buf.toString(); +ThreadDumpHelper helper = new ThreadDumpHelper(); +Stream.of(threadMXBean.dumpAllThreads(true, true)).forEach(helper::addThread); +return helper.dumpAsString(); +} + +static class ThreadDumpHelper { + +private final StringBuilder buf = new StringBuilder(32 * 1024); + +private ThreadDumpHelper() { +} + +private void addThread(ThreadInfo ti) { +buf.append('\n'); +quote(ti.getThreadName()).append(" [tid=").append(ti.getThreadId()).append(" state=") +.append(ti.getThreadState()); + +if (ti.getLockName() != null) { +buf.append(" lock=").append(ti.getLockName()); +if (ti.getLockOwnerName() != null) { +buf.append(" lockOwner="); +quote(ti.getLockOwnerName()).append(" (tid=").append(ti.getLockOwnerId()); +} +} +if (ti.isSuspended()) { +buf.append(" suspended=true"); +} +buf.append("]\n"); +MutableInt depth = new MutableInt(); +for (StackTraceElement frame : ti.getStackTrace()) { +int thisDepth = depth.getAndIncrement(); +buf.append("\tat ").append(frame).append('\n'); +Stream.of(ti.getLockedMonitors()).filter(m -> m.getLockedStackDepth() == thisDepth) +.forEach(this::output); +} +LockInfo[] lockedSynchronizers = ti.getLockedSynchronizers(); +if (lockedSynchronizers.length > 0) { +buf.append("\n\tLocked synchronizers:\n"); +Stream.of(lockedSynchronizers).forEachOrdered(this::output); +} +} + +private StringBuilder quote(Object quotable) { +return buf.append('"').append(quotable).append('"'); +} + +private StringBuilder output(MonitorInfo info) { +return buf.append("\t- <").append("locked ").append(info).append(">\n"); +} + +private StringBuilder output(LockInfo info) { +return buf.append("\t- ").append("").append(info).
[15/34] asterixdb git commit: [NO ISSUE] Incremental cleanup of deprecated exception ctors
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/6453ecf2/asterixdb/asterix-om/src/main/java/org/apache/asterix/dataflow/data/nontagged/printers/PrintTools.java -- diff --git a/asterixdb/asterix-om/src/main/java/org/apache/asterix/dataflow/data/nontagged/printers/PrintTools.java b/asterixdb/asterix-om/src/main/java/org/apache/asterix/dataflow/data/nontagged/printers/PrintTools.java index 8d05f0f..b8e1b43 100644 --- a/asterixdb/asterix-om/src/main/java/org/apache/asterix/dataflow/data/nontagged/printers/PrintTools.java +++ b/asterixdb/asterix-om/src/main/java/org/apache/asterix/dataflow/data/nontagged/printers/PrintTools.java @@ -44,7 +44,7 @@ public class PrintTools { gCalInstance.getExtendStringRepUntilField(chrononTime, 0, ps, GregorianCalendarSystem.Fields.YEAR, GregorianCalendarSystem.Fields.DAY, false); } catch (IOException e) { -throw new HyracksDataException(e); +throw HyracksDataException.create(e); } } @@ -55,7 +55,7 @@ public class PrintTools { gCalInstance.getExtendStringRepUntilField(chrononTime, 0, ps, GregorianCalendarSystem.Fields.YEAR, GregorianCalendarSystem.Fields.MILLISECOND, true); } catch (IOException e) { -throw new HyracksDataException(e); +throw HyracksDataException.create(e); } } @@ -106,7 +106,7 @@ public class PrintTools { ps.print("S"); } } catch (IOException e) { -throw new HyracksDataException(e); +throw HyracksDataException.create(e); } } @@ -139,7 +139,7 @@ public class PrintTools { ps.print("M"); } } catch (IOException e) { -throw new HyracksDataException(e); +throw HyracksDataException.create(e); } } @@ -202,7 +202,7 @@ public class PrintTools { ps.print("S"); } } catch (IOException e) { -throw new HyracksDataException(e); +throw HyracksDataException.create(e); } } @@ -213,7 +213,7 @@ public class PrintTools { gCalInstance.getExtendStringRepUntilField(time, 0, ps, GregorianCalendarSystem.Fields.HOUR, GregorianCalendarSystem.Fields.MILLISECOND, true); } catch (IOException e) { -throw new HyracksDataException(e); +throw HyracksDataException.create(e); } } http://git-wip-us.apache.org/repos/asf/asterixdb/blob/6453ecf2/asterixdb/asterix-om/src/main/java/org/apache/asterix/dataflow/data/nontagged/printers/adm/ABinaryHexPrinterFactory.java -- diff --git a/asterixdb/asterix-om/src/main/java/org/apache/asterix/dataflow/data/nontagged/printers/adm/ABinaryHexPrinterFactory.java b/asterixdb/asterix-om/src/main/java/org/apache/asterix/dataflow/data/nontagged/printers/adm/ABinaryHexPrinterFactory.java index b5ac617..a2a3e3b 100644 --- a/asterixdb/asterix-om/src/main/java/org/apache/asterix/dataflow/data/nontagged/printers/adm/ABinaryHexPrinterFactory.java +++ b/asterixdb/asterix-om/src/main/java/org/apache/asterix/dataflow/data/nontagged/printers/adm/ABinaryHexPrinterFactory.java @@ -44,7 +44,7 @@ public class ABinaryHexPrinterFactory implements IPrinterFactory { HexPrinter.printHexString(b, start, validLength, ps); ps.print("\")"); } catch (IOException e) { -throw new HyracksDataException(e); +throw HyracksDataException.create(e); } }; http://git-wip-us.apache.org/repos/asf/asterixdb/blob/6453ecf2/asterixdb/asterix-om/src/main/java/org/apache/asterix/dataflow/data/nontagged/printers/adm/AInt16PrinterFactory.java -- diff --git a/asterixdb/asterix-om/src/main/java/org/apache/asterix/dataflow/data/nontagged/printers/adm/AInt16PrinterFactory.java b/asterixdb/asterix-om/src/main/java/org/apache/asterix/dataflow/data/nontagged/printers/adm/AInt16PrinterFactory.java index a7830e3..222e896 100644 --- a/asterixdb/asterix-om/src/main/java/org/apache/asterix/dataflow/data/nontagged/printers/adm/AInt16PrinterFactory.java +++ b/asterixdb/asterix-om/src/main/java/org/apache/asterix/dataflow/data/nontagged/printers/adm/AInt16PrinterFactory.java @@ -36,7 +36,7 @@ public class AInt16PrinterFactory implements IPrinterFactory { try { WriteValueTools.writeInt(AInt16SerializerDeserializer.getShort(b, s + 1), ps); } catch (IOException e) { -throw new HyracksDataException(e); +throw HyracksDataException.create(e); } }; http://git-wip-us.apache.org/repos/asf/asterixdb/blob/6453ecf2/asterixdb/asterix-om/src/main/java/org/apache/asterix/dataflow/data/nontagged/pr
[26/34] asterixdb git commit: [NO ISSUE][FAIL] Remove Not Serializable Field From ACIDException
[NO ISSUE][FAIL] Remove Not Serializable Field From ACIDException - user model changes: no - storage format changes: no - interface changes: no Details: - Remove not serializable ITransactionContext from ACIDException to avoid serialization error when reporting errors to CC. - Properly handle InterruptedException in lock manager. - Remove unneeded WaitInterruptedException class. Change-Id: Iee054a432b3e618579c3bc418175deab6abfb965 Reviewed-on: https://asterix-gerrit.ics.uci.edu/2559 Sonar-Qube: Jenkins Tested-by: Jenkins Contrib: Jenkins Integration-Tests: Jenkins Reviewed-by: Till Westmann Project: http://git-wip-us.apache.org/repos/asf/asterixdb/repo Commit: http://git-wip-us.apache.org/repos/asf/asterixdb/commit/366edf8b Tree: http://git-wip-us.apache.org/repos/asf/asterixdb/tree/366edf8b Diff: http://git-wip-us.apache.org/repos/asf/asterixdb/diff/366edf8b Branch: refs/heads/release-0.9.4-pre-rc Commit: 366edf8b222ec84d1993d606a2bcd965f03e84fb Parents: b1843b2 Author: Murtadha Hubail Authored: Mon Apr 2 14:12:19 2018 +0300 Committer: Murtadha Hubail Committed: Mon Apr 2 08:42:12 2018 -0700 -- .../common/exceptions/ACIDException.java| 21 -- .../service/locking/ConcurrentLockManager.java | 6 ++-- .../locking/WaitInterruptedException.java | 30 .../service/locking/LockManagerUnitTest.java| 3 +- 4 files changed, 6 insertions(+), 54 deletions(-) -- http://git-wip-us.apache.org/repos/asf/asterixdb/blob/366edf8b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/exceptions/ACIDException.java -- diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/exceptions/ACIDException.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/exceptions/ACIDException.java index 77634eb..9775b45 100644 --- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/exceptions/ACIDException.java +++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/exceptions/ACIDException.java @@ -18,8 +18,6 @@ */ package org.apache.asterix.common.exceptions; -import org.apache.asterix.common.transactions.ITransactionContext; - /** * Represents an exception related to an unexpected behavior that prevents the * system from supporting ACID guarantees. The exception contains the @@ -30,25 +28,6 @@ import org.apache.asterix.common.transactions.ITransactionContext; public class ACIDException extends RuntimeException { private static final long serialVersionUID = -8855848112541877323L; -private ITransactionContext txnContext; - -public ITransactionContext getTxnContext() { -return txnContext; -} - -public void setTxnContext(ITransactionContext txnContext) { -this.txnContext = txnContext; -} - -public ACIDException(ITransactionContext txnContext, String message) { -super(message); -this.txnContext = txnContext; -} - -public ACIDException(ITransactionContext txnContext, String message, Throwable cause) { -super(message, cause); -this.txnContext = txnContext; -} public ACIDException(String message, Throwable cause) { super(message, cause); http://git-wip-us.apache.org/repos/asf/asterixdb/blob/366edf8b/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/service/locking/ConcurrentLockManager.java -- diff --git a/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/service/locking/ConcurrentLockManager.java b/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/service/locking/ConcurrentLockManager.java index 726f95c..c91d233 100644 --- a/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/service/locking/ConcurrentLockManager.java +++ b/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/service/locking/ConcurrentLockManager.java @@ -146,7 +146,8 @@ public class ConcurrentLockManager implements ILockManager, ILifeCycleComponent } } } catch (InterruptedException e) { -throw new WaitInterruptedException(txnContext, "interrupted", e); +Thread.currentThread().interrupt(); +throw new ACIDException(e); } finally { group.releaseLatch(); } @@ -371,7 +372,8 @@ public class ConcurrentLockManager implements ILockManager, ILifeCycleComponent } } } catch (InterruptedException e) { -throw new WaitInterruptedException(txnContext, "interrupted", e); +Thread.currentThread().interrupt();
[22/34] asterixdb git commit: [ASTERIXDB-2348][COMP] Incorrect result with distinct aggregate
[ASTERIXDB-2348][COMP] Incorrect result with distinct aggregate - user model changes: no - storage format changes: no - interface changes: no Details: - Account for micro group by/distinct by operators in EnforceStructuralPropertiesRule Change-Id: I0f440046d6cdb2dc0302c5aa4c01ffacb703ee73 Reviewed-on: https://asterix-gerrit.ics.uci.edu/2544 Tested-by: Jenkins Contrib: Jenkins Integration-Tests: Jenkins Reviewed-by: Till Westmann Project: http://git-wip-us.apache.org/repos/asf/asterixdb/repo Commit: http://git-wip-us.apache.org/repos/asf/asterixdb/commit/cba9ec76 Tree: http://git-wip-us.apache.org/repos/asf/asterixdb/tree/cba9ec76 Diff: http://git-wip-us.apache.org/repos/asf/asterixdb/diff/cba9ec76 Branch: refs/heads/release-0.9.4-pre-rc Commit: cba9ec76ceaefdfe9a3eafa695f01d739bb74c18 Parents: cc9257e Author: Dmitry Lychagin Authored: Wed Mar 28 18:23:24 2018 -0700 Committer: Till Westmann Committed: Thu Mar 29 22:45:31 2018 -0700 -- .../issue2348/issue2348.1.ddl.sqlpp | 30 .../issue2348/issue2348.2.update.sqlpp | 27 ++ .../issue2348/issue2348.3.query.sqlpp | 26 + .../aggregate-sql/issue2348/issue2348.3.adm | 3 ++ .../resources/runtimets/testsuite_sqlpp.xml | 5 .../rules/EnforceStructuralPropertiesRule.java | 14 + 6 files changed, 99 insertions(+), 6 deletions(-) -- http://git-wip-us.apache.org/repos/asf/asterixdb/blob/cba9ec76/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/aggregate-sql/issue2348/issue2348.1.ddl.sqlpp -- diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/aggregate-sql/issue2348/issue2348.1.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/aggregate-sql/issue2348/issue2348.1.ddl.sqlpp new file mode 100644 index 000..a50d20b --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/aggregate-sql/issue2348/issue2348.1.ddl.sqlpp @@ -0,0 +1,30 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +drop dataverse test if exists; +create dataverse test; + +use test; + +create type test.TestType as +{ + c_id: string +}; + +create dataset t1(TestType) primary key c_id; http://git-wip-us.apache.org/repos/asf/asterixdb/blob/cba9ec76/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/aggregate-sql/issue2348/issue2348.2.update.sqlpp -- diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/aggregate-sql/issue2348/issue2348.2.update.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/aggregate-sql/issue2348/issue2348.2.update.sqlpp new file mode 100644 index 000..ec72ef5 --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/aggregate-sql/issue2348/issue2348.2.update.sqlpp @@ -0,0 +1,27 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +use test; + +insert into t1 select value t from [ + {"c_id":"01","c_b":true, "c_d":900,"c_i":99}, + {"c_id":"02","c_b":true, "c_d":800,"c_i":88}, + {"c_id":"03","c_b":true, "c_d":700,"c_i":77}, + {"c_id":"05","c_b":false,"c_d"
[31/34] asterixdb git commit: [NO ISSUE][CLUSTER] Poll CC Registration Response
[NO ISSUE][CLUSTER] Poll CC Registration Response - user model changes: no - storage format changes: no - interface changes: no Details: - Check for CC registration response while waiting to avoid waiting forever. Change-Id: I6e2d5ea0ef315d90784a58ee2748a422812b4a2f Reviewed-on: https://asterix-gerrit.ics.uci.edu/2569 Reviewed-by: Michael Blow Sonar-Qube: Jenkins Tested-by: Jenkins Contrib: Jenkins Integration-Tests: Jenkins Project: http://git-wip-us.apache.org/repos/asf/asterixdb/repo Commit: http://git-wip-us.apache.org/repos/asf/asterixdb/commit/29a0cee0 Tree: http://git-wip-us.apache.org/repos/asf/asterixdb/tree/29a0cee0 Diff: http://git-wip-us.apache.org/repos/asf/asterixdb/diff/29a0cee0 Branch: refs/heads/release-0.9.4-pre-rc Commit: 29a0cee0092f80676a93e005c9099af19bcf7db1 Parents: da479dc Author: Murtadha Hubail Authored: Thu Apr 5 01:00:41 2018 +0300 Committer: Murtadha Hubail Committed: Wed Apr 4 16:51:15 2018 -0700 -- .../main/java/org/apache/hyracks/control/nc/CcConnection.java | 5 - 1 file changed, 4 insertions(+), 1 deletion(-) -- http://git-wip-us.apache.org/repos/asf/asterixdb/blob/29a0cee0/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/CcConnection.java -- diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/CcConnection.java b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/CcConnection.java index dce7d35..627e972 100644 --- a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/CcConnection.java +++ b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/CcConnection.java @@ -33,6 +33,7 @@ import org.apache.logging.log4j.Logger; public class CcConnection { private static final Logger LOGGER = LogManager.getLogger(); +private static final long REGISTRATION_RESPONSE_POLL_PERIOD = TimeUnit.SECONDS.toMillis(1); private final IClusterController ccs; private boolean registrationPending; @@ -64,7 +65,9 @@ public class CcConnection { registrationPending = true; ccs.registerNode(nodeRegistration, registrationId); try { -InvokeUtil.runWithTimeout(this::wait, () -> !registrationPending, 2, TimeUnit.MINUTES); +InvokeUtil.runWithTimeout(() -> { +this.wait(REGISTRATION_RESPONSE_POLL_PERIOD); // NOSONAR while loop in timeout call +}, () -> !registrationPending, 1, TimeUnit.MINUTES); } catch (Exception e) { registrationException = e; }
[19/34] asterixdb git commit: [NO ISSUE][REPL] Use String.compareTo To Sort Replicated Files
[NO ISSUE][REPL] Use String.compareTo To Sort Replicated Files - user model changes: no - storage format changes: no - interface changes: no - Use String.compareTo to sort replicated files instead of custom comparator. Change-Id: I45c106d5cff6e07d16a9a7ce286a2ac563a80338 Reviewed-on: https://asterix-gerrit.ics.uci.edu/2547 Reviewed-by: Michael Blow Sonar-Qube: Jenkins Integration-Tests: Jenkins Tested-by: Jenkins Contrib: Jenkins Project: http://git-wip-us.apache.org/repos/asf/asterixdb/repo Commit: http://git-wip-us.apache.org/repos/asf/asterixdb/commit/6ee05df4 Tree: http://git-wip-us.apache.org/repos/asf/asterixdb/tree/6ee05df4 Diff: http://git-wip-us.apache.org/repos/asf/asterixdb/diff/6ee05df4 Branch: refs/heads/release-0.9.4-pre-rc Commit: 6ee05df4c2a0c3a234d95d0375a18bf3aef496f2 Parents: 6ee96a9 Author: Murtadha Hubail Authored: Thu Mar 29 09:00:17 2018 +0300 Committer: Murtadha Hubail Committed: Thu Mar 29 01:40:16 2018 -0700 -- .../replication/sync/ReplicaFilesSynchronizer.java| 14 ++ 1 file changed, 2 insertions(+), 12 deletions(-) -- http://git-wip-us.apache.org/repos/asf/asterixdb/blob/6ee05df4/asterixdb/asterix-replication/src/main/java/org/apache/asterix/replication/sync/ReplicaFilesSynchronizer.java -- diff --git a/asterixdb/asterix-replication/src/main/java/org/apache/asterix/replication/sync/ReplicaFilesSynchronizer.java b/asterixdb/asterix-replication/src/main/java/org/apache/asterix/replication/sync/ReplicaFilesSynchronizer.java index 0d97a7a..583f33d 100644 --- a/asterixdb/asterix-replication/src/main/java/org/apache/asterix/replication/sync/ReplicaFilesSynchronizer.java +++ b/asterixdb/asterix-replication/src/main/java/org/apache/asterix/replication/sync/ReplicaFilesSynchronizer.java @@ -18,12 +18,9 @@ */ package org.apache.asterix.replication.sync; -import static org.apache.asterix.common.utils.StorageConstants.METADATA_FILE_NAME; - import java.io.IOException; import java.nio.ByteBuffer; import java.nio.channels.SocketChannel; -import java.util.Comparator; import java.util.HashSet; import java.util.List; import java.util.Set; @@ -43,13 +40,6 @@ import org.apache.asterix.transaction.management.resource.PersistentLocalResourc */ public class ReplicaFilesSynchronizer { -private static final Comparator REPLICATED_FILES_COMPARATOR = (file, anotherFile) -> { -if (file.endsWith(METADATA_FILE_NAME) && !anotherFile.endsWith(METADATA_FILE_NAME)) { -return -1; -} -return file.compareTo(anotherFile); -}; - private final PartitionReplica replica; private final INcApplicationContext appCtx; @@ -89,8 +79,8 @@ public class ReplicaFilesSynchronizer { private void replicateMissingFiles(List files) { final FileSynchronizer sync = new FileSynchronizer(appCtx, replica); -// sort files to ensure index metadata files are replicated first -files.sort(REPLICATED_FILES_COMPARATOR); +// sort files to ensure index metadata files starting with "." are replicated first +files.sort(String::compareTo); files.forEach(sync::replicate); }
[17/34] asterixdb git commit: [NO ISSUE][RT] Inherit Closeable in IFrameOperationCallback
[NO ISSUE][RT] Inherit Closeable in IFrameOperationCallback - user model changes: no - storage format changes: no - interface changes: yes Details: - Primary upsert operator uses an IFrameOperationCallback to perform per frame operations. In this change, IFrameOperationCallback extends Closeable and allow the callback to release resources on the close of the pipeline. Change-Id: Iaacd6538c27d5498868256616a793e0ebcec3729 Reviewed-on: https://asterix-gerrit.ics.uci.edu/2536 Sonar-Qube: Jenkins Tested-by: Jenkins Contrib: Jenkins Integration-Tests: Jenkins Reviewed-by: Murtadha Hubail Project: http://git-wip-us.apache.org/repos/asf/asterixdb/repo Commit: http://git-wip-us.apache.org/repos/asf/asterixdb/commit/bf41e4cb Tree: http://git-wip-us.apache.org/repos/asf/asterixdb/tree/bf41e4cb Diff: http://git-wip-us.apache.org/repos/asf/asterixdb/diff/bf41e4cb Branch: refs/heads/release-0.9.4-pre-rc Commit: bf41e4cb3cfd659f6b00e783532d470ce2b7281e Parents: 6453ecf Author: Abdullah Alamoudi Authored: Mon Mar 26 21:10:08 2018 -0700 Committer: abdullah alamoudi Committed: Wed Mar 28 12:31:01 2018 -0700 -- .../common/context/BaseOperationTracker.java| 4 .../NoOpFrameOperationCallbackFactory.java | 7 +++ .../LSMPrimaryUpsertOperatorNodePushable.java | 22 +++- .../am/common/api/IIndexDataflowHelper.java | 5 - .../lsm/common/api/IFrameOperationCallback.java | 5 +++-- 5 files changed, 30 insertions(+), 13 deletions(-) -- http://git-wip-us.apache.org/repos/asf/asterixdb/blob/bf41e4cb/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/BaseOperationTracker.java -- diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/BaseOperationTracker.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/BaseOperationTracker.java index 9ec13ef..b7af0b6 100644 --- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/BaseOperationTracker.java +++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/BaseOperationTracker.java @@ -74,4 +74,8 @@ public class BaseOperationTracker implements ITransactionOperationTracker { dsInfo.untouch(); dsInfo.getIndexes().get(resourceId).untouch(); } + +public DatasetInfo getDatasetInfo() { +return dsInfo; +} } http://git-wip-us.apache.org/repos/asf/asterixdb/blob/bf41e4cb/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/dataflow/NoOpFrameOperationCallbackFactory.java -- diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/dataflow/NoOpFrameOperationCallbackFactory.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/dataflow/NoOpFrameOperationCallbackFactory.java index 5e3879f..8f28752 100644 --- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/dataflow/NoOpFrameOperationCallbackFactory.java +++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/dataflow/NoOpFrameOperationCallbackFactory.java @@ -18,6 +18,8 @@ */ package org.apache.asterix.common.dataflow; +import java.io.IOException; + import org.apache.hyracks.api.context.IHyracksTaskContext; import org.apache.hyracks.api.exceptions.HyracksDataException; import org.apache.hyracks.storage.am.lsm.common.api.IFrameOperationCallback; @@ -43,5 +45,10 @@ public class NoOpFrameOperationCallbackFactory implements IFrameOperationCallbac public void frameCompleted() throws HyracksDataException { // No Op } + +@Override +public void close() throws IOException { +// No Op +} } } http://git-wip-us.apache.org/repos/asf/asterixdb/blob/bf41e4cb/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/operators/LSMPrimaryUpsertOperatorNodePushable.java -- diff --git a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/operators/LSMPrimaryUpsertOperatorNodePushable.java b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/operators/LSMPrimaryUpsertOperatorNodePushable.java index e58a7db..68053d3 100644 --- a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/operators/LSMPrimaryUpsertOperatorNodePushable.java +++ b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/operators/LSMPrimaryUpsertOperatorNodePushable.java @@ -40,6 +40,7 @@ import org.apache.hyracks.api.dataflow.value.IMissingWriter; import org.apache.hyracks.api.dataflow.value.IMissingWriterFactory; import org.apache.hyracks.api.dataflow.value.RecordDescriptor; import org.apache.h
[27/34] asterixdb git commit: [ASTERIXDB-2352][FUN] Incorrect leap year handling in duration arithmetic
[ASTERIXDB-2352][FUN] Incorrect leap year handling in duration arithmetic - user model changes: no - storage format changes: no - interface changes: no Details: - DurationArithmeticOperations.addDuration() incorrectly handled leap year Change-Id: I01e1417f4704d72a4650ec4f2fa257f044a20a09 Reviewed-on: https://asterix-gerrit.ics.uci.edu/2561 Reviewed-by: Till Westmann Sonar-Qube: Jenkins Tested-by: Jenkins Contrib: Jenkins Integration-Tests: Jenkins Project: http://git-wip-us.apache.org/repos/asf/asterixdb/repo Commit: http://git-wip-us.apache.org/repos/asf/asterixdb/commit/8db70084 Tree: http://git-wip-us.apache.org/repos/asf/asterixdb/tree/8db70084 Diff: http://git-wip-us.apache.org/repos/asf/asterixdb/diff/8db70084 Branch: refs/heads/release-0.9.4-pre-rc Commit: 8db70084f0593983af6ed877ccf50553dec4649c Parents: 366edf8 Author: Dmitry Lychagin Authored: Mon Apr 2 15:36:46 2018 -0700 Committer: Dmitry Lychagin Committed: Mon Apr 2 19:24:52 2018 -0700 -- .../om/base/temporal/DurationArithmeticOperations.java | 8 1 file changed, 4 insertions(+), 4 deletions(-) -- http://git-wip-us.apache.org/repos/asf/asterixdb/blob/8db70084/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/base/temporal/DurationArithmeticOperations.java -- diff --git a/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/base/temporal/DurationArithmeticOperations.java b/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/base/temporal/DurationArithmeticOperations.java index a50adc6..26c3fb3 100644 --- a/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/base/temporal/DurationArithmeticOperations.java +++ b/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/base/temporal/DurationArithmeticOperations.java @@ -85,13 +85,13 @@ public class DurationArithmeticOperations { boolean isLeapYear = GREG_CAL.isLeapYear(year); if (isLeapYear) { -if (day > GregorianCalendarSystem.DAYS_OF_MONTH_ORDI[month - 1]) { -day = GregorianCalendarSystem.DAYS_OF_MONTH_ORDI[month - 1]; -} -} else { if (day > GregorianCalendarSystem.DAYS_OF_MONTH_LEAP[month - 1]) { day = GregorianCalendarSystem.DAYS_OF_MONTH_LEAP[month - 1]; } +} else { +if (day > GregorianCalendarSystem.DAYS_OF_MONTH_ORDI[month - 1]) { +day = GregorianCalendarSystem.DAYS_OF_MONTH_ORDI[month - 1]; +} } return GREG_CAL.getChronon(year, month, day, hour, min, sec, ms, 0) + dayTimeDuration;
[20/34] asterixdb git commit: [NO ISSUE][COMP] Check logging level before writing a log message
[NO ISSUE][COMP] Check logging level before writing a log message - user model changes: no - storage format changes: no - interface changes: no Details: - Cleanup: Check logging level before writing to the logger Change-Id: Ib126397c390f0d32436a80a306981106afa2a794 Reviewed-on: https://asterix-gerrit.ics.uci.edu/2543 Tested-by: Jenkins Contrib: Jenkins Integration-Tests: Jenkins Reviewed-by: Michael Blow Project: http://git-wip-us.apache.org/repos/asf/asterixdb/repo Commit: http://git-wip-us.apache.org/repos/asf/asterixdb/commit/8e4b57bf Tree: http://git-wip-us.apache.org/repos/asf/asterixdb/tree/8e4b57bf Diff: http://git-wip-us.apache.org/repos/asf/asterixdb/diff/8e4b57bf Branch: refs/heads/release-0.9.4-pre-rc Commit: 8e4b57bf05779c700fe582b16692bd59939d0205 Parents: 6ee05df Author: Dmitry Lychagin Authored: Wed Mar 28 15:46:30 2018 -0700 Committer: Dmitry Lychagin Committed: Thu Mar 29 12:57:57 2018 -0700 -- .../operators/logical/IntersectOperator.java| 8 ++- .../visitors/FDsAndEquivClassesVisitor.java | 8 ++- .../visitors/LogicalPropertiesVisitor.java | 2 +- .../core/rewriter/base/HeuristicOptimizer.java | 2 +- .../rules/EnforceStructuralPropertiesRule.java | 63 +--- .../subplan/IntroduceGroupByForSubplanRule.java | 4 +- .../algebricks/rewriter/util/JoinUtils.java | 13 ++-- .../util/AlgebricksHyracksIntegrationUtil.java | 10 +++- 8 files changed, 74 insertions(+), 36 deletions(-) -- http://git-wip-us.apache.org/repos/asf/asterixdb/blob/8e4b57bf/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/IntersectOperator.java -- diff --git a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/IntersectOperator.java b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/IntersectOperator.java index 8a06ec4..c2e4541 100644 --- a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/IntersectOperator.java +++ b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/IntersectOperator.java @@ -155,9 +155,11 @@ public class IntersectOperator extends AbstractLogicalOperator { Object expectedType = expected.getVarType(expectedVariables.get(i)); Object actualType = actual.getVarType(actualVariables.get(i)); if (!expectedType.equals(actualType)) { -AlgebricksConfig.ALGEBRICKS_LOGGER -.warn("Type of two variables are not equal." + expectedVariables.get(i) + " is of type: " -+ expectedType + actualVariables.get(i) + " is of type: " + actualType); +if (AlgebricksConfig.ALGEBRICKS_LOGGER.isWarnEnabled()) { +AlgebricksConfig.ALGEBRICKS_LOGGER +.warn("Type of two variables are not equal." + expectedVariables.get(i) + " is of type: " ++ expectedType + actualVariables.get(i) + " is of type: " + actualType); +} } } } http://git-wip-us.apache.org/repos/asf/asterixdb/blob/8e4b57bf/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/visitors/FDsAndEquivClassesVisitor.java -- diff --git a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/visitors/FDsAndEquivClassesVisitor.java b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/visitors/FDsAndEquivClassesVisitor.java index 39d522f..2d5780d 100644 --- a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/visitors/FDsAndEquivClassesVisitor.java +++ b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/visitors/FDsAndEquivClassesVisitor.java @@ -314,9 +314,11 @@ public class FDsAndEquivClassesVisitor implements ILogicalOperatorVisitor>>> Group-by list changed from " + GroupByOperator.veListToString(gByList) + " to " -+ GroupByOperator.veListToString(newGbyList) + ".\n"); +if (AlgebricksConfig.ALGEBRICKS_LOGGER.isDebugEnabled()) { +AlgebricksConfig.ALGEBRICKS_LOGGER +.debug(" Group-by list changed from " + GroupByOperator.veListToString(gB
[28/34] asterixdb git commit: [NO ISSUE][TRACE] Cleanups for performance tracer
[NO ISSUE][TRACE] Cleanups for performance tracer - user model changes: no - storage format changes: no - interface changes: yes Details: - Move all Trace categories into a single place. - Cleanup the interface and add some javadocs. - Performance optimizations when logger is used. - Add a new storage log for last record in. Change-Id: Ib65edd6e7093dfd6f94ef750c484501b92c7a8a0 Reviewed-on: https://asterix-gerrit.ics.uci.edu/2558 Tested-by: Jenkins Contrib: Jenkins Integration-Tests: Jenkins Reviewed-by: abdullah alamoudi Project: http://git-wip-us.apache.org/repos/asf/asterixdb/repo Commit: http://git-wip-us.apache.org/repos/asf/asterixdb/commit/5ff35c98 Tree: http://git-wip-us.apache.org/repos/asf/asterixdb/tree/5ff35c98 Diff: http://git-wip-us.apache.org/repos/asf/asterixdb/diff/5ff35c98 Branch: refs/heads/release-0.9.4-pre-rc Commit: 5ff35c98bd58d02a7b18ad886d09bb15f54885ff Parents: 8db7008 Author: Abdullah Alamoudi Authored: Mon Apr 2 22:31:42 2018 -0700 Committer: abdullah alamoudi Committed: Tue Apr 3 00:09:51 2018 -0700 -- .../operators/FeedMetaStoreNodePushable.java| 3 +- .../LSMPrimaryUpsertOperatorNodePushable.java | 39 + .../control/nc/NodeControllerService.java | 3 +- .../impls/AbstractLSMIndexOperationContext.java | 5 ++- .../am/lsm/common/impls/TracedIOOperation.java | 18 .../util/trace/ITraceCategoryRegistry.java | 21 ++--- .../util/trace/TraceCategoryRegistry.java | 46 +++- .../apache/hyracks/util/trace/TraceUtils.java | 35 +++ .../org/apache/hyracks/util/trace/Tracer.java | 14 -- 9 files changed, 134 insertions(+), 50 deletions(-) -- http://git-wip-us.apache.org/repos/asf/asterixdb/blob/5ff35c98/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/operators/FeedMetaStoreNodePushable.java -- diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/operators/FeedMetaStoreNodePushable.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/operators/FeedMetaStoreNodePushable.java index b9cfac4..94ae75c 100644 --- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/operators/FeedMetaStoreNodePushable.java +++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/operators/FeedMetaStoreNodePushable.java @@ -42,6 +42,7 @@ import org.apache.hyracks.dataflow.common.comm.io.FrameTupleAccessor; import org.apache.hyracks.dataflow.common.utils.TaskUtil; import org.apache.hyracks.dataflow.std.base.AbstractUnaryInputUnaryOutputOperatorNodePushable; import org.apache.hyracks.util.trace.ITracer; +import org.apache.hyracks.util.trace.TraceUtils; import org.apache.logging.log4j.Level; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -108,7 +109,7 @@ public class FeedMetaStoreNodePushable extends AbstractUnaryInputUnaryOutputOper this.recordDescProvider = recordDescProvider; this.opDesc = feedMetaOperatorDescriptor; tracer = ctx.getJobletContext().getServiceContext().getTracer(); -traceCategory = tracer.getRegistry().get("Process-Frame"); +traceCategory = tracer.getRegistry().get(TraceUtils.STORAGE); } @Override http://git-wip-us.apache.org/repos/asf/asterixdb/blob/5ff35c98/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/operators/LSMPrimaryUpsertOperatorNodePushable.java -- diff --git a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/operators/LSMPrimaryUpsertOperatorNodePushable.java b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/operators/LSMPrimaryUpsertOperatorNodePushable.java index 68053d3..c767157 100644 --- a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/operators/LSMPrimaryUpsertOperatorNodePushable.java +++ b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/operators/LSMPrimaryUpsertOperatorNodePushable.java @@ -21,6 +21,8 @@ package org.apache.asterix.runtime.operators; import java.io.DataOutput; import java.io.IOException; import java.nio.ByteBuffer; +import java.text.SimpleDateFormat; +import java.util.Date; import org.apache.asterix.common.api.INcApplicationContext; import org.apache.asterix.common.dataflow.LSMIndexUtil; @@ -68,6 +70,9 @@ import org.apache.hyracks.storage.am.lsm.common.impls.LSMTreeIndexAccessor; import org.apache.hyracks.storage.common.IIndexAccessParameters; import org.apache.hyracks.storage.common.IIndexCursor; import org.apache.hyracks.storage.common.MultiComparator; +import org.apache.hyracks.util.trace.ITracer; +import org.apache.hyracks.util.trace.IT
[24/34] asterixdb git commit: [ASTERIXDB-2351][COMP] Allow '+' after exponent indicator in double literals
[ASTERIXDB-2351][COMP] Allow '+' after exponent indicator in double literals - user model changes: no - storage format changes: no - interface changes: no Details: - Allow '+' sign after exponent indicator when parsing double literals in scientific notation Change-Id: I1f27c0db786269cce90ef43463df1a398fd1e5cd Reviewed-on: https://asterix-gerrit.ics.uci.edu/2552 Sonar-Qube: Jenkins Tested-by: Jenkins Integration-Tests: Jenkins Contrib: Jenkins Reviewed-by: Till Westmann Project: http://git-wip-us.apache.org/repos/asf/asterixdb/repo Commit: http://git-wip-us.apache.org/repos/asf/asterixdb/commit/ee8526bd Tree: http://git-wip-us.apache.org/repos/asf/asterixdb/tree/ee8526bd Diff: http://git-wip-us.apache.org/repos/asf/asterixdb/diff/ee8526bd Branch: refs/heads/release-0.9.4-pre-rc Commit: ee8526bd97e6abdb9b1021b337b04d2d12f8876b Parents: 0918229 Author: Dmitry Lychagin Authored: Fri Mar 30 14:21:01 2018 -0700 Committer: Dmitry Lychagin Committed: Fri Mar 30 16:07:16 2018 -0700 -- .../queries/numeric/scientific/scientific.1.query.aql | 4 +++- .../queries_sqlpp/numeric/scientific/scientific.1.query.sqlpp | 6 +- .../runtimets/results/numeric/scientific/scientific.1.adm | 2 +- asterixdb/asterix-lang-aql/src/main/javacc/AQL.jj | 6 +++--- asterixdb/asterix-lang-sqlpp/src/main/javacc/SQLPP.jj | 6 +++--- 5 files changed, 15 insertions(+), 9 deletions(-) -- http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ee8526bd/asterixdb/asterix-app/src/test/resources/runtimets/queries/numeric/scientific/scientific.1.query.aql -- diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries/numeric/scientific/scientific.1.query.aql b/asterixdb/asterix-app/src/test/resources/runtimets/queries/numeric/scientific/scientific.1.query.aql index d668b0b..2ae391b 100644 --- a/asterixdb/asterix-app/src/test/resources/runtimets/queries/numeric/scientific/scientific.1.query.aql +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries/numeric/scientific/scientific.1.query.aql @@ -17,5 +17,7 @@ * under the License. */ -[2e5, 2e-5, .2e2, .2e-2, 0.5e3, 0.5e-3, 3.2e5, 3.2e-5] +let $v1 := [2e5, 2e+5, 2e-5, .2e2, .2e+2, .2e-2, 0.5e3, 0.5e+3, 0.5e-3, 3.2e5, 3.2e+5, 3.2e-5 ] +let $v2 := [2E5, 2E+5, 2E-5, .2E2, .2E+2, .2E-2, 0.5E3, 0.5E+3, 0.5E-3, 3.2E5, 3.2E+5, 3.2E-5 ] +return { "t1": $v1, "t2": $v2 } http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ee8526bd/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/numeric/scientific/scientific.1.query.sqlpp -- diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/numeric/scientific/scientific.1.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/numeric/scientific/scientific.1.query.sqlpp index d668b0b..2bfe15f 100644 --- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/numeric/scientific/scientific.1.query.sqlpp +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/numeric/scientific/scientific.1.query.sqlpp @@ -17,5 +17,9 @@ * under the License. */ -[2e5, 2e-5, .2e2, .2e-2, 0.5e3, 0.5e-3, 3.2e5, 3.2e-5] +{ + "t1": [2e5, 2e+5, 2e-5, .2e2, .2e+2, .2e-2, 0.5e3, 0.5e+3, 0.5e-3, 3.2e5, 3.2e+5, 3.2e-5 ], + "t2": [2E5, 2E+5, 2E-5, .2E2, .2E+2, .2E-2, 0.5E3, 0.5E+3, 0.5E-3, 3.2E5, 3.2E+5, 3.2E-5 ] +} + http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ee8526bd/asterixdb/asterix-app/src/test/resources/runtimets/results/numeric/scientific/scientific.1.adm -- diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/numeric/scientific/scientific.1.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/numeric/scientific/scientific.1.adm index f65c2fe..1a2e70f 100644 --- a/asterixdb/asterix-app/src/test/resources/runtimets/results/numeric/scientific/scientific.1.adm +++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/numeric/scientific/scientific.1.adm @@ -1 +1 @@ -[ 20.0, 2.0E-5, 20.0, 0.002, 500.0, 5.0E-4, 32.0, 3.2E-5 ] \ No newline at end of file +{ "t1": [ 20.0, 20.0, 2.0E-5, 20.0, 20.0, 0.002, 500.0, 500.0, 5.0E-4, 32.0, 32.0, 3.2E-5 ], "t2": [ 20.0, 20.0, 2.0E-5, 20.0, 20.0, 0.002, 500.0, 500.0, 5.0E-4, 32.0, 32.0, 3.2E-5 ] } \ No newline at end of file http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ee8526bd/asterixdb/asterix-lang-aql/src/main/javacc/AQL.jj -- diff --git a/asterixdb/asterix-lang-aql/src/main/javacc/AQL.jj b/asterixdb/asterix-lang-aql/src/main/javacc/AQL.jj index 74fe907..2d87