This is an automated email from the ASF dual-hosted git repository. bertty pushed a commit to branch python-platform in repository https://gitbox.apache.org/repos/asf/incubator-wayang.git
commit cdd1147db0a173113f1026832d695118d4b53eea Author: Bertty Contreras-Rojas <[email protected]> AuthorDate: Thu Apr 7 11:55:35 2022 +0200 [WAYANG-#8] Structure folder changes Signed-off-by: bertty <[email protected]> --- python/src/pywy/core/__init__.py | 18 ++++++++++++++++ .../src/pywy/{platforms/basic => core}/channel.py | 0 .../src/pywy/{platforms/basic => core}/executor.py | 0 .../src/pywy/{platforms/basic => core}/mapping.py | 2 +- python/src/pywy/{platforms/basic => core}/plan.py | 4 ++-- .../src/pywy/{platforms/basic => core}/platform.py | 0 .../src/pywy/{platforms/basic => core}/plugin.py | 6 +++--- .../pywy/{platforms/basic => core}/translator.py | 6 +++--- python/src/pywy/dataquanta.py | 8 +++---- python/src/pywy/graph/graphtypes.py | 2 +- python/src/pywy/operators/__init__.py | 13 +++++++++++ python/src/pywy/{wayangplan => operators}/base.py | 2 +- python/src/pywy/{wayangplan => operators}/sink.py | 2 +- .../src/pywy/{wayangplan => operators}/source.py | 2 +- python/src/pywy/{wayangplan => operators}/unary.py | 2 +- python/src/pywy/platforms/basic/__init__.py | 0 python/src/pywy/platforms/python/channels.py | 2 +- .../pywy/platforms/python/execution/executor.py | 6 +++--- python/src/pywy/platforms/python/mappings.py | 2 +- .../python/operators/PyExecutionOperator.py | 2 +- .../platforms/python/operators/PyFilterOperator.py | 2 +- .../python/operators/PyTextFileSinkOperator.py | 2 +- .../python/operators/PyTextFileSourceOperator.py | 2 +- .../src/pywy/platforms/python/platform/platform.py | 2 +- python/src/pywy/platforms/python/plugin/plugin.py | 4 ++-- python/src/pywy/plugins.py | 4 ++-- python/src/pywy/test.py | 25 +++++++++++----------- python/src/pywy/wayangplan/__init__.py | 13 ----------- 28 files changed, 76 insertions(+), 57 deletions(-) diff --git a/python/src/pywy/core/__init__.py b/python/src/pywy/core/__init__.py new file mode 100644 index 00000000..90da09e4 --- /dev/null +++ b/python/src/pywy/core/__init__.py @@ -0,0 +1,18 @@ +from pywy.core.channel import Channel, ChannelDescriptor +from pywy.core.executor import Executor +from pywy.core.mapping import Mapping +from pywy.core.plan import PywyPlan +from pywy.core.platform import Platform +from pywy.core.plugin import Plugin +from pywy.core.translator import Translator + +__ALL__ = [ + Channel, + ChannelDescriptor, + Executor, + Mapping, + PywyPlan, + Platform, + Plugin, + Translator +] \ No newline at end of file diff --git a/python/src/pywy/platforms/basic/channel.py b/python/src/pywy/core/channel.py similarity index 100% rename from python/src/pywy/platforms/basic/channel.py rename to python/src/pywy/core/channel.py diff --git a/python/src/pywy/platforms/basic/executor.py b/python/src/pywy/core/executor.py similarity index 100% rename from python/src/pywy/platforms/basic/executor.py rename to python/src/pywy/core/executor.py diff --git a/python/src/pywy/platforms/basic/mapping.py b/python/src/pywy/core/mapping.py similarity index 89% rename from python/src/pywy/platforms/basic/mapping.py rename to python/src/pywy/core/mapping.py index 9749ebdd..db377928 100644 --- a/python/src/pywy/platforms/basic/mapping.py +++ b/python/src/pywy/core/mapping.py @@ -1,5 +1,5 @@ from typing import Dict -from pywy.wayangplan.base import PywyOperator +from pywy.operators.base import PywyOperator class Mapping: mappings: Dict[str, type] diff --git a/python/src/pywy/platforms/basic/plan.py b/python/src/pywy/core/plan.py similarity index 95% rename from python/src/pywy/platforms/basic/plan.py rename to python/src/pywy/core/plan.py index 1e8b16ac..53d06fb5 100644 --- a/python/src/pywy/platforms/basic/plan.py +++ b/python/src/pywy/core/plan.py @@ -2,8 +2,8 @@ from typing import ( Iterable, Set ) from pywy.graph.graph import WayangGraph from pywy.graph.graphtypes import ( NodeOperator, WGraphOfVec, NodeVec ) -from pywy.wayangplan.sink import SinkOperator -from pywy.platforms.basic.plugin import Plugin +from pywy.operators.sink import SinkOperator +from pywy.core.plugin import Plugin class PywyPlan: diff --git a/python/src/pywy/platforms/basic/platform.py b/python/src/pywy/core/platform.py similarity index 100% rename from python/src/pywy/platforms/basic/platform.py rename to python/src/pywy/core/platform.py diff --git a/python/src/pywy/platforms/basic/plugin.py b/python/src/pywy/core/plugin.py similarity index 83% rename from python/src/pywy/platforms/basic/plugin.py rename to python/src/pywy/core/plugin.py index 838d88a4..b6f84d88 100644 --- a/python/src/pywy/platforms/basic/plugin.py +++ b/python/src/pywy/core/plugin.py @@ -1,8 +1,8 @@ from typing import Set -from pywy.platforms.basic.executor import Executor -from pywy.platforms.basic.platform import Platform -from pywy.platforms.basic.mapping import Mapping +from pywy.core.executor import Executor +from pywy.core.platform import Platform +from pywy.core.mapping import Mapping class Plugin: """ diff --git a/python/src/pywy/platforms/basic/translator.py b/python/src/pywy/core/translator.py similarity index 90% rename from python/src/pywy/platforms/basic/translator.py rename to python/src/pywy/core/translator.py index 6031f7a2..2719f426 100644 --- a/python/src/pywy/platforms/basic/translator.py +++ b/python/src/pywy/core/translator.py @@ -1,7 +1,7 @@ from pywy.graph.graphtypes import ( WGraphOfVec, NodeVec ) -from pywy.platforms.basic.plugin import Plugin -from pywy.platforms.basic.plan import PywyPlan -from pywy.platforms.basic.mapping import Mapping +from pywy.core.plugin import Plugin +from pywy.core.plan import PywyPlan +from pywy.core.mapping import Mapping class Translator: diff --git a/python/src/pywy/dataquanta.py b/python/src/pywy/dataquanta.py index 14019423..3d465c29 100644 --- a/python/src/pywy/dataquanta.py +++ b/python/src/pywy/dataquanta.py @@ -1,10 +1,10 @@ from typing import Set -from pywy.platforms.basic.translator import Translator +from pywy.core import Translator from pywy.types import ( GenericTco, Predicate, Function, FlatmapFunction, IterableO ) -from pywy.wayangplan import * -from pywy.platforms.basic.plan import PywyPlan -from pywy.platforms.basic.plugin import Plugin +from pywy.operators import * +from pywy.core import PywyPlan +from pywy.core import Plugin class WayangContext: """ diff --git a/python/src/pywy/graph/graphtypes.py b/python/src/pywy/graph/graphtypes.py index 26664edd..2c014263 100644 --- a/python/src/pywy/graph/graphtypes.py +++ b/python/src/pywy/graph/graphtypes.py @@ -1,7 +1,7 @@ from typing import ( Iterable, List ) from pywy.graph.graph import ( GraphNode, WayangGraph ) -from pywy.wayangplan.base import PywyOperator +from pywy.operators.base import PywyOperator class NodeOperator(GraphNode[PywyOperator]): diff --git a/python/src/pywy/operators/__init__.py b/python/src/pywy/operators/__init__.py new file mode 100644 index 00000000..68306811 --- /dev/null +++ b/python/src/pywy/operators/__init__.py @@ -0,0 +1,13 @@ +from pywy.operators.base import PywyOperator +from pywy.operators.sink import TextFileSink +from pywy.operators.source import TextFileSource +from pywy.operators.unary import FilterOperator, MapOperator, FlatmapOperator +# +__ALL__= [ + PywyOperator, + TextFileSink, + TextFileSource, + FilterOperator, +# MapOperator, +# FlatmapOperator +] \ No newline at end of file diff --git a/python/src/pywy/wayangplan/base.py b/python/src/pywy/operators/base.py similarity index 97% rename from python/src/pywy/wayangplan/base.py rename to python/src/pywy/operators/base.py index ebe2e2ff..26e8bf9b 100644 --- a/python/src/pywy/wayangplan/base.py +++ b/python/src/pywy/operators/base.py @@ -1,5 +1,5 @@ from typing import ( TypeVar, Optional, List, Set ) -from pywy.platforms.basic.channel import ChannelDescriptor, Channel +from pywy.core import ChannelDescriptor, Channel class PywyOperator: diff --git a/python/src/pywy/wayangplan/sink.py b/python/src/pywy/operators/sink.py similarity index 89% rename from python/src/pywy/wayangplan/sink.py rename to python/src/pywy/operators/sink.py index 3cd26adc..e3ddd7d8 100644 --- a/python/src/pywy/wayangplan/sink.py +++ b/python/src/pywy/operators/sink.py @@ -1,7 +1,7 @@ from typing import Any from pywy.types import GenericTco -from pywy.wayangplan.base import PywyOperator +from pywy.operators.base import PywyOperator class SinkOperator(PywyOperator): diff --git a/python/src/pywy/wayangplan/source.py b/python/src/pywy/operators/source.py similarity index 89% rename from python/src/pywy/wayangplan/source.py rename to python/src/pywy/operators/source.py index 1439c108..42af498e 100644 --- a/python/src/pywy/wayangplan/source.py +++ b/python/src/pywy/operators/source.py @@ -1,4 +1,4 @@ -from pywy.wayangplan.base import PywyOperator +from pywy.operators.base import PywyOperator class SourceUnaryOperator(PywyOperator): diff --git a/python/src/pywy/wayangplan/unary.py b/python/src/pywy/operators/unary.py similarity index 96% rename from python/src/pywy/wayangplan/unary.py rename to python/src/pywy/operators/unary.py index 725159e3..5b4b8b0a 100644 --- a/python/src/pywy/wayangplan/unary.py +++ b/python/src/pywy/operators/unary.py @@ -1,5 +1,5 @@ from itertools import chain -from pywy.wayangplan.base import PywyOperator +from pywy.operators.base import PywyOperator from pywy.types import ( GenericTco, GenericUco, diff --git a/python/src/pywy/platforms/basic/__init__.py b/python/src/pywy/platforms/basic/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/python/src/pywy/platforms/python/channels.py b/python/src/pywy/platforms/python/channels.py index 5f0b7a7a..dc976bb6 100644 --- a/python/src/pywy/platforms/python/channels.py +++ b/python/src/pywy/platforms/python/channels.py @@ -1,5 +1,5 @@ from typing import ( Iterable, Callable ) -from pywy.platforms.basic.channel import ( Channel, ChannelDescriptor ) +from pywy.core import (Channel, ChannelDescriptor) class PyIteratorChannel(Channel): diff --git a/python/src/pywy/platforms/python/execution/executor.py b/python/src/pywy/platforms/python/execution/executor.py index fa17ae8f..93bba462 100644 --- a/python/src/pywy/platforms/python/execution/executor.py +++ b/python/src/pywy/platforms/python/execution/executor.py @@ -1,9 +1,9 @@ from typing import List from pywy.graph.graphtypes import WGraphOfOperator, NodeOperator -from pywy.platforms.basic.channel import Channel -from pywy.platforms.basic.executor import Executor -from pywy.platforms.basic.plan import PywyPlan +from pywy.core import Channel +from pywy.core import Executor +from pywy.core import PywyPlan from pywy.platforms.python.operators.PyExecutionOperator import PyExecutionOperator diff --git a/python/src/pywy/platforms/python/mappings.py b/python/src/pywy/platforms/python/mappings.py index 1c2783aa..e0411beb 100644 --- a/python/src/pywy/platforms/python/mappings.py +++ b/python/src/pywy/platforms/python/mappings.py @@ -1,4 +1,4 @@ -from pywy.platforms.basic.mapping import Mapping +from pywy.core import Mapping from pywy.platforms.python.operators import * diff --git a/python/src/pywy/platforms/python/operators/PyExecutionOperator.py b/python/src/pywy/platforms/python/operators/PyExecutionOperator.py index a9d4ebd5..718e9c49 100644 --- a/python/src/pywy/platforms/python/operators/PyExecutionOperator.py +++ b/python/src/pywy/platforms/python/operators/PyExecutionOperator.py @@ -1,4 +1,4 @@ -from pywy.wayangplan.base import PywyOperator +from pywy.operators.base import PywyOperator from pywy.platforms.python.channels import Channel class PyExecutionOperator(PywyOperator): diff --git a/python/src/pywy/platforms/python/operators/PyFilterOperator.py b/python/src/pywy/platforms/python/operators/PyFilterOperator.py index 7d6a503c..e9e4e385 100644 --- a/python/src/pywy/platforms/python/operators/PyFilterOperator.py +++ b/python/src/pywy/platforms/python/operators/PyFilterOperator.py @@ -1,5 +1,5 @@ from typing import Set -from pywy.wayangplan.unary import FilterOperator +from pywy.operators.unary import FilterOperator from pywy.platforms.python.operators.PyExecutionOperator import PyExecutionOperator from pywy.platforms.python.channels import ( Channel, diff --git a/python/src/pywy/platforms/python/operators/PyTextFileSinkOperator.py b/python/src/pywy/platforms/python/operators/PyTextFileSinkOperator.py index 5a67cfaf..20a23f73 100644 --- a/python/src/pywy/platforms/python/operators/PyTextFileSinkOperator.py +++ b/python/src/pywy/platforms/python/operators/PyTextFileSinkOperator.py @@ -1,5 +1,5 @@ from typing import Set -from pywy.wayangplan.sink import TextFileSink +from pywy.operators.sink import TextFileSink from pywy.platforms.python.operators.PyExecutionOperator import PyExecutionOperator from pywy.platforms.python.channels import ( Channel, diff --git a/python/src/pywy/platforms/python/operators/PyTextFileSourceOperator.py b/python/src/pywy/platforms/python/operators/PyTextFileSourceOperator.py index bac1f844..4ca43d75 100644 --- a/python/src/pywy/platforms/python/operators/PyTextFileSourceOperator.py +++ b/python/src/pywy/platforms/python/operators/PyTextFileSourceOperator.py @@ -1,5 +1,5 @@ from typing import Set -from pywy.wayangplan.source import TextFileSource +from pywy.operators.source import TextFileSource from pywy.platforms.python.operators.PyExecutionOperator import PyExecutionOperator from pywy.platforms.python.channels import ( Channel, diff --git a/python/src/pywy/platforms/python/platform/platform.py b/python/src/pywy/platforms/python/platform/platform.py index c4faddb0..e2aa3fed 100644 --- a/python/src/pywy/platforms/python/platform/platform.py +++ b/python/src/pywy/platforms/python/platform/platform.py @@ -1,4 +1,4 @@ -from pywy.platforms.basic.platform import Platform +from pywy.core.platform import Platform class PythonPlatform(Platform): diff --git a/python/src/pywy/platforms/python/plugin/plugin.py b/python/src/pywy/platforms/python/plugin/plugin.py index 34814638..ead1a859 100644 --- a/python/src/pywy/platforms/python/plugin/plugin.py +++ b/python/src/pywy/platforms/python/plugin/plugin.py @@ -1,7 +1,7 @@ -from pywy.platforms.basic.executor import Executor +from pywy.core import Executor from pywy.platforms.python.execution.executor import PyExecutor from pywy.platforms.python.platform import PythonPlatform -from pywy.platforms.basic.plugin import Plugin +from pywy.core import Plugin from pywy.platforms.python.mappings import PywyOperatorMappings diff --git a/python/src/pywy/plugins.py b/python/src/pywy/plugins.py index 1e55e310..cad2f356 100644 --- a/python/src/pywy/plugins.py +++ b/python/src/pywy/plugins.py @@ -1,5 +1,5 @@ -from pywy.platforms.basic.platform import Platform -from pywy.platforms.basic.plugin import Plugin +from pywy.core.platform import Platform +from pywy.core import Plugin from pywy.platforms.python.plugin import PythonPlugin # define the basic plugins that can be used diff --git a/python/src/pywy/test.py b/python/src/pywy/test.py index 2de508e6..9a2b544f 100644 --- a/python/src/pywy/test.py +++ b/python/src/pywy/test.py @@ -1,8 +1,5 @@ -from pywy.platforms.basic.platform import Platform from pywy.dataquanta import WayangContext -from pywy.platforms.python.channels import Channel -from pywy.plugins import java, spark, python -from pywy.wayangplan.unary import * +from pywy.plugins import python # p = Platform("nana") # print("LALA "+str(p)) @@ -15,13 +12,12 @@ from pywy.wayangplan.unary import * # # print(str(WayangContext().register(java, spark))) -from pywy.types import Predicate, getTypePredicate # # predicate : Predicate = lambda x: x % 2 == 0 # getTypePredicate(predicate) - +import time def pre(a:str): - return len(a) > 3 + return "six" in a # # def func(s:str) -> int: # return len(s) @@ -30,11 +26,16 @@ def pre(a:str): # for x in range(i): # yield str(x) -fileop = WayangContext()\ - .register(python)\ - .textFile("/Users/bertty/databloom/blossom/python/resources/test.input")\ - .filter(pre)\ - .storeTextFile("/Users/bertty/databloom/blossom/python/resources/test.output") +for index in range(0, 1): + print(index) + tic = time.perf_counter() + fileop = WayangContext()\ + .register(python)\ + .textFile("/Users/bertty/databloom/blossom/python/resources/tmp"+str(index))\ + .filter(pre)\ + .storeTextFile("/Users/bertty/databloom/blossom/python/resources/out"+str(index)) + toc = time.perf_counter() + print(f"Downloaded the tutorial in {toc - tic:0.4f} seconds") # filterop: FilterOperator = fileop.filter(pre).getOperator() # #fop_pre = filterop.getWrapper() diff --git a/python/src/pywy/wayangplan/__init__.py b/python/src/pywy/wayangplan/__init__.py deleted file mode 100644 index b0c87a10..00000000 --- a/python/src/pywy/wayangplan/__init__.py +++ /dev/null @@ -1,13 +0,0 @@ -from pywy.wayangplan.base import PywyOperator -from pywy.wayangplan.sink import TextFileSink -from pywy.wayangplan.source import TextFileSource -from pywy.wayangplan.unary import FilterOperator, MapOperator, FlatmapOperator -# -__ALL__= [ - PywyOperator, - TextFileSink, - TextFileSource, - FilterOperator, -# MapOperator, -# FlatmapOperator -] \ No newline at end of file
