diff --git a/contrib/tajo-python/README.md b/contrib/tajo-python/README.md new file mode 100644 index 0000000000..ab3a5d9bbd --- /dev/null +++ b/contrib/tajo-python/README.md @@ -0,0 +1,5 @@ +# python-tajo-rest-client +Python Tajo Rest Client Module + +# requirements +pip install httplib2 diff --git a/contrib/tajo-python/setup.py b/contrib/tajo-python/setup.py new file mode 100644 index 0000000000..63a38c8189 --- /dev/null +++ b/contrib/tajo-python/setup.py @@ -0,0 +1,42 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from setuptools import setup, find_packages +import sys + +DESCRIPTION = """python tajo rest client +""" + +install_requires = ["httplib2"] + +setup( + name="tajo-rest-client", + version="0.0.1", + description="a Python implementation of Tajo Rest Client", + long_description=DESCRIPTION, + url='http://github.com/charsyam/python-tajo-rest-client/', + author='DaeMyung Kang', + author_email='charsyam@gmail.com', + classifiers=[ + 'Intended Audience :: Developers', + 'License :: OSI Approved :: Apache Software License', + 'Operating System :: OS Independent', + 'Programming Language :: Python', + 'Topic :: Software Development :: Libraries :: Python Modules'], + packages=find_packages('src'), + package_dir={'': 'src'}, + install_requires=install_requires, + test_suite='', +) diff --git a/contrib/tajo-python/src/tajo/__init__.py b/contrib/tajo-python/src/tajo/__init__.py new file mode 100644 index 0000000000..ae1e83eeb3 --- /dev/null +++ b/contrib/tajo-python/src/tajo/__init__.py @@ -0,0 +1,14 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/contrib/tajo-python/src/tajo/base.py b/contrib/tajo-python/src/tajo/base.py new file mode 100644 index 0000000000..e465f9d885 --- /dev/null +++ b/contrib/tajo-python/src/tajo/base.py @@ -0,0 +1,86 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from tajo.error import * +from tajo.py3 import httplib, PY3 + +try: + import simplejson as json +except ImportError: + import json + +class TajoObject(object): + def __init__(self): + pass + + @staticmethod + def create(headers, content): + raise NotImplementedMethodError("object create error") + + +class TajoRequest(object): + object_cls = None + ok_status = [httplib.OK] + + def __init__(self): + pass + + def method(self): + return "GET" + + def uri(self): + raise NotImplementedMethodError("uri") + + def params(self): + raise NotImplementedMethodError("params") + + def headers(self): + raise NotImplementedMethodError("headers") + + def object_cls(self): + if self.object_cls is None: + raise NotImplementedMethodError("cls") + + return self.object_cls + + def check_status(self, headers, contents): + status = int(headers["status"]) + if status not in self.ok_status: + msg = status + if PY3: + contents = contents.decode('utf-8') + + c = json.loads(contents) + if 'message' in c: + msg = "%s %s"%(status, c["message"]) + + if headers["status"][0] == '4': + raise InvalidRequestError(msg) + if headers["status"][0] == '5': + raise InternalError(msg) + + def request(self, conn): + headers, contents = conn._request(self.method(), self.uri(), self.params()) + self.check_status(headers, contents) + return self.object_cls.create(headers, contents) + + +class TajoPostRequest(TajoRequest): + def method(self): + return "POST" + +class TajoDeleteRequest(TajoRequest): + def method(self): + return "DELETE" diff --git a/contrib/tajo-python/src/tajo/client.py b/contrib/tajo-python/src/tajo/client.py new file mode 100644 index 0000000000..426d88ec4f --- /dev/null +++ b/contrib/tajo-python/src/tajo/client.py @@ -0,0 +1,166 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from tajo.session import TajoSessionRequest, TajoSession +from tajo.query import TajoQueryRequest, TajoQuery +from tajo.queryid import QueryId +from tajo.queries import TajoQueriesRequest +from tajo.querystatus import TajoQueryStatusRequest, TajoQueryStatus +from tajo.resultsetinfo import TajoResultSetInfoRequest, TajoResultSetInfo +from tajo.resultset import TajoMemoryResultSetRequest, TajoMemoryResultSet +from tajo.fetchresultset import TajoFetchResultSet +from tajo.connection import TajoConnection +from tajo.cluster import TajoCluster, TajoClusterRequest +from tajo.querystate import QueryState +from tajo.database import TajoDatabasesRequest, TajoDatabases, TajoDatabaseRequest, TajoDatabase +from tajo.database import TajoCreateDatabaseRequest, TajoDeleteDatabaseRequest +from tajo.tables import TajoTablesRequest, TajoTableRequest +from tajo.functions import TajoFunctionsRequest +from tajo.error import * + +import time + +class TajoClient(object): + def __init__(self, base = 'http://127.0.0.1:26880/rest/', + username = 'tajo', database = 'default', + rowNum = 1024): + self.clear() + self.base = base + self.username = username + self.database = database + self.rowNum = rowNum + self.conn = TajoConnection(self.base) + self.session = self.create_session() + + def clear(self): + self.base = None + self.username = "tajo" + self.database = "default" + self.rowNum = 1024 + self.conn = None + self.session = None + + def create_session(self): + request = TajoSessionRequest(self.username, self.database) + self.session = request.request(self.conn) + self.conn.add_header("X-Tajo-Session", str(self.session)) + + def execute_query(self, query): + req = TajoQueryRequest(query) + return req.request(self.conn) + + def is_null_query_id(self, query_id): + ret = False + + if query_id.query_id == QueryId.NULL_QUERY_ID: + ret = True + + return ret + + def queries(self): + req = TajoQueriesRequest(self.database) + return req.request(self.conn) + + def query_status(self, query_id): + if query_id.completed: + return TajoQueryStatus(query_id.status) + elif self.is_null_query_id(query_id): + raise NullQueryIdError() + + req = TajoQueryStatusRequest(query_id, self.base) + return req.request(self.conn) + + def query_resultset_info(self, query_id): + if self.is_null_query_id(query_id): + raise NullQueryIdError() + + if query_id.completed: + return TajoResultSetInfo(None, query_id.schema) + + req = TajoResultSetInfoRequest(query_id, self.base) + return req.request(self.conn) + + def query_resultset(self, resultsetinfo, count = 100): + req = TajoMemoryResultSetRequest(resultsetinfo, self.base, count) + return req.request(self.conn) + + def fetch(self, query_id, fetch_row_num = 100): + resultset_info = self.query_resultset_info(query_id) + return TajoFetchResultSet(self, query_id, resultset_info, fetch_row_num) + + def create_nullresultset(self, queryId): + return TajoMemoryResultSet(None, True, 0, 0, None) + + def execute_query_wait_result(self, query): + query_id = self.execute_query(query) + if self.is_null_query_id(query_id): + return self.create_nullresultset(query_id) + + status = self.query_status(query_id) + while self.is_query_complete(status.state) == False: + time.sleep(0.1) + status = query_status(query_id) + + if status.state == QueryState.QUERY_SUCCEEDED: + return self.fetch(query_id) + + return self.create_nullresultset(query_id) + + def is_query_waiting_for_schedule(self, state): + return state == QueryState.QUERY_NOT_ASSIGNED or \ + state == QueryState.QUERY_MASTER_INIT or \ + state == QueryState.QUERY_MASTER_LAUNCHED + + def is_query_inited(self, state): + return state == QueryState.QUERY_NEW + + def is_query_running(self, state): + return self.is_query_inited(state) or (state == QueryState.QUERY_RUNNING) + + def is_query_complete(self, state): + return self.is_query_waiting_for_schedule(state) == False and \ + self.is_query_running(state) == False + + def cluster_info(self): + req = TajoClusterRequest() + return req.request(self.conn) + + def databases(self): + req = TajoDatabasesRequest() + return req.request(self.conn) + + def database_info(self, database_name): + req = TajoDatabaseRequest(database_name) + return req.request(self.conn) + + def create_database(self, database_name): + req = TajoCreateDatabaseRequest(database_name) + return req.request(self.conn) + + def delete_database(self, database_name): + req = TajoDeleteDatabaseRequest(database_name) + return req.request(self.conn) + + def tables(self, database_name): + req = TajoTablesRequest(database_name) + return req.request(self.conn) + + def table(self, database_name, table_name): + req = TajoTableRequest(database_name, table_name) + return req.request(self.conn) + + def functions(self): + req = TajoFunctionsRequest() + return req.request(self.conn) diff --git a/contrib/tajo-python/src/tajo/cluster.py b/contrib/tajo-python/src/tajo/cluster.py new file mode 100644 index 0000000000..5f0a5f869c --- /dev/null +++ b/contrib/tajo-python/src/tajo/cluster.py @@ -0,0 +1,57 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from tajo.error import InvalidStatusError +from tajo.base import TajoRequest, TajoObject +from tajo.py3 import httplib, PY3 + +try: + import simplejson as json +except ImportError: + import json + +class TajoCluster(TajoObject): + def __init__(self, objs): + self.objs = objs + + def __repr__(self): + return str(self.objs) + + @staticmethod + def create(headers, content): + if PY3: + content = content.decode('utf-8') + + return TajoCluster(content) + + +class TajoClusterRequest(TajoRequest): + object_cls = TajoCluster + ok_status = [httplib.OK] + + def __init__(self): + self.objs = None + + def uri(self): + return "cluster" + + def headers(self): + return None + + def params(self): + return None + + def cls(self): + return self.object_cls diff --git a/contrib/tajo-python/src/tajo/column.py b/contrib/tajo-python/src/tajo/column.py new file mode 100644 index 0000000000..0c28d98dec --- /dev/null +++ b/contrib/tajo-python/src/tajo/column.py @@ -0,0 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +class Column: + def __init__(self, name, datatype, length = 1): + self.name = name + self.datatype = datatype.upper() + self.length = length diff --git a/contrib/tajo-python/src/tajo/connection.py b/contrib/tajo-python/src/tajo/connection.py new file mode 100644 index 0000000000..5c5146451c --- /dev/null +++ b/contrib/tajo-python/src/tajo/connection.py @@ -0,0 +1,40 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import httplib2 as Http +import json + +class TajoConnection(object): + def __init__(self, base): + self.base = base + self.request = Http.Http() + self.common_headers = {'Content-Type': 'application/json'} + + def add_header(self, key, value): + self.common_headers[key] = value + + def post(self, uri, body, headers = None): + return self._request("POST", uri, body, headers) + + def get(self, uri, body, headers = None): + return self._request("GET", uri, body, headers) + + def _request(self, method, uri, body = None, headers = None): + url = "%s/%s"%(self.base, uri) + aheaders = self.common_headers.copy() + if headers is not None: + aheaders.update(headers) + + return self.request.request(url, method, body=json.dumps(body), headers=aheaders) diff --git a/contrib/tajo-python/src/tajo/database.py b/contrib/tajo-python/src/tajo/database.py new file mode 100644 index 0000000000..f939c9ea35 --- /dev/null +++ b/contrib/tajo-python/src/tajo/database.py @@ -0,0 +1,155 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from tajo.error import InvalidStatusError +from tajo.base import TajoRequest, TajoObject, TajoPostRequest, TajoDeleteRequest +from tajo.py3 import httplib, PY3 + +try: + import simplejson as json +except ImportError: + import json + +class TajoCreateDatabase(TajoObject): + @staticmethod + def create(headers, content): + return True; + + +class TajoDeleteDatabase(TajoObject): + @staticmethod + def create(headers, content): + return True; + + +class TajoDatabaseName(TajoObject): + def __init__(self, name): + self.database_name = name + + def __repr__(self): + return self.database_name + + +class TajoDatabase(TajoObject): + def __init__(self, objs): + self.objs = objs + + def __repr__(self): + return self.objs["name"] + + @staticmethod + def create(headers, content): + if PY3: + content = content.decode('utf-8') + + data = json.loads(content) + return TajoDatabase(data) + + +class TajoDatabases(TajoObject): + @staticmethod + def create(headers, content): + if PY3: + content = content.decode('utf-8') + + data = json.loads(content) + databases = [] + for database in data["databases"]: + databases.append(TajoDatabaseName(database)) + + return databases + + +class TajoDatabaseRequest(TajoRequest): + object_cls = TajoDatabase + ok_status = [httplib.OK] + + def __init__(self, database): + self.database_name = database.database_name + + def uri(self): + return "databases/%s"%(self.database_name) + + def headers(self): + return None + + def params(self): + return None + + def cls(self): + return self.object_cls + + +class TajoDatabasesRequest(TajoRequest): + object_cls = TajoDatabases + ok_status = [httplib.OK] + + def __init__(self): + pass + + def uri(self): + return "databases" + + def headers(self): + return None + + def params(self): + return None + + def cls(self): + return self.object_cls + + +class TajoCreateDatabaseRequest(TajoPostRequest): + object_cls = TajoCreateDatabase + ok_status = [httplib.CREATED] + + def __init__(self, database_name): + self.database_name = database_name + + def uri(self): + return "databases" + + def headers(self): + return None + + def params(self): + payload = { + "databaseName": self.database_name + } + return payload + + def cls(self): + return self.object_cls + + +class TajoDeleteDatabaseRequest(TajoDeleteRequest): + object_cls = TajoDeleteDatabase + ok_status = [httplib.OK] + + def __init__(self, database_name): + self.database_name = database_name + + def uri(self): + return "databases/%s"%(self.database_name) + + def headers(self): + return None + + def params(self): + return None + + def cls(self): + return self.object_cls diff --git a/contrib/tajo-python/src/tajo/datatypes.py b/contrib/tajo-python/src/tajo/datatypes.py new file mode 100644 index 0000000000..c6bfac0202 --- /dev/null +++ b/contrib/tajo-python/src/tajo/datatypes.py @@ -0,0 +1,31 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +class TajoDataTypes: + BOOLEAN = "BOOLEAN" + INT1 = "INT1" + INT2 = "INT2" + INT4 = "INT4" + INT8 = "INT8" + FLOAT4 = "FLOAT4" + FLOAT8 = "FLOAT8" + TEXT = "TEXT" + DATE = "DATE" + TIME = "TIME" + TIMESTAMP = "TIMESTAMP" + BLOB = "BLOB" + INET4 = "INET4" + INET6 = "INET6" + CHAR = "CHAR" diff --git a/contrib/tajo-python/src/tajo/error.py b/contrib/tajo-python/src/tajo/error.py new file mode 100644 index 0000000000..204a59375c --- /dev/null +++ b/contrib/tajo-python/src/tajo/error.py @@ -0,0 +1,86 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +class TajoClientError(Exception): + """The base class for other Tajo Client exceptions""" + + def __init__(self, value): + super(TajoClientError, self).__init__(value) + self.value = value + + def __str__(self): + return self.__repr__() + + def __repr__(self): + return ("") + +class NotImplementedMethodError(TajoClientError): + """The NotImplementedMethodError exceptions""" + + def __init__(self, value): + super(NotImplementedMethodError, self).__init__(value) + self.value = value + + def __repr__(self): + return ("") + +class InvalidStatusError(TajoClientError): + """Invalid Status Error exceptions""" + + def __init__(self, value): + super(InvalidStatusError, self).__init__(value) + self.value = value + + def __repr__(self): + return ("") + +class InvalidRequestError(TajoClientError): + """Invalid Request Error exceptions""" + + def __init__(self, value): + super(InvalidRequestError, self).__init__(value) + self.value = value + + def __repr__(self): + return ("") + +class NotFoundError(TajoClientError): + """NotFound Error exceptions""" + + def __init__(self, value): + super(NotFoundError, self).__init__(value) + self.value = value + + def __repr__(self): + return ("") + +class NullQueryIdError(TajoClientError): + """NotFound Error exceptions""" + + def __init__(self): + super(NullQueryIdError, self).__init__() + + def __repr__(self): + return ("") + +class InternalError(TajoClientError): + """NotFound Error exceptions""" + + def __init__(self, value): + super(InternalError, self).__init__(value) + self.value = value + + def __repr__(self): + return ("") diff --git a/contrib/tajo-python/src/tajo/fetchresultset.py b/contrib/tajo-python/src/tajo/fetchresultset.py new file mode 100644 index 0000000000..7c12e12019 --- /dev/null +++ b/contrib/tajo-python/src/tajo/fetchresultset.py @@ -0,0 +1,68 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from tajo.resultsetbase import ResultSetBase +from tajo.resultset import TajoMemoryResultSetRequest, TajoMemoryResultSet + +class TajoFetchResultSet(ResultSetBase): + def __init__(self, client, query_id, resultset_info, fetch_row_num = 100): + super(TajoFetchResultSet, self).__init__() + self.fetch_row_num = fetch_row_num + self.query_id = query_id + self.resultset_info = resultset_info + self.finished = False + self.resultset = None + self.schema = resultset_info.schema() + self.offset = -1 + self.client = client + + def is_finished(self): + return self.finished + + def fetch(self): + return self.client.query_resultset(self.resultset_info, self.fetch_row_num) + + def next_tuple(self): + if self.is_finished() is True: + return None + + t = None + if self.resultset is not None: + self.resultset.next() + t = self.resultset.current_tuple() + + if self.resultset is None or t is None: + if self.resultset is None or (self.resultset is not None and self.resultset.eos == False): + self.resultset = self.fetch() + + if self.resultset is None: + self.finished = True + return None + + if self.offset == -1: + self.offset = 0 + + self.offset += self.fetch_row_num + + self.resultset.next() + t = self.resultset.current_tuple() + + if t is None: + if self.resultset is not None: + self.resultset = None + + self.finished = True + + return t diff --git a/contrib/tajo-python/src/tajo/functions.py b/contrib/tajo-python/src/tajo/functions.py new file mode 100644 index 0000000000..3f981a55fa --- /dev/null +++ b/contrib/tajo-python/src/tajo/functions.py @@ -0,0 +1,67 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from tajo.error import InvalidStatusError +from tajo.base import TajoRequest, TajoObject +from tajo.py3 import httplib, PY3 + +try: + import simplejson as json +except ImportError: + import json + +class TajoFunction: + def __init__(self, function_type, name, param_types, return_type): + self.name = name + self.param_types = param_types + self.return_type = return_type["type"] + self.function_type = function_type + + def __repr__(self): + return "%s:%s"%(self.name, self.return_type) + + +class TajoFunctions(TajoObject): + @staticmethod + def create(headers, content): + if PY3: + content = content.decode('utf-8') + + functions = json.loads(content) + results = [] + for f in functions: + results.append(TajoFunction(f["functionType"], f["name"], f["paramTypes"], f["returnType"])) + + return results; + + +class TajoFunctionsRequest(TajoRequest): + object_cls = TajoFunctions + ok_status = [httplib.OK] + + def __init__(self): + pass + + def uri(self): + return "functions" + + def headers(self): + return None + + def params(self): + return None + + def cls(self): + return self.object_cls diff --git a/contrib/tajo-python/src/tajo/py3.py b/contrib/tajo-python/src/tajo/py3.py new file mode 100644 index 0000000000..f7a66c03e2 --- /dev/null +++ b/contrib/tajo-python/src/tajo/py3.py @@ -0,0 +1,219 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Libcloud Python 2.x and 3.x compatibility layer +# Some methods below are taken from Django PYK3 port which is licensed under 3 +# clause BSD license +# https://bitbucket.org/loewis/django-3k + +from __future__ import absolute_import + +import sys +import types + +try: + from lxml import etree as ET +except ImportError: + from xml.etree import ElementTree as ET + +PY2 = sys.version_info[0] == 2 +PY3 = sys.version_info[0] == 3 +PY2_pre_25 = PY2 and sys.version_info < (2, 5) +PY2_pre_26 = PY2 and sys.version_info < (2, 6) +PY2_pre_27 = PY2 and sys.version_info < (2, 7) +PY2_pre_279 = PY2 and sys.version_info < (2, 7, 9) +PY3_pre_32 = PY3 and sys.version_info < (3, 2) + +PY2 = False +PY25 = False +PY26 = False +PY27 = False +PY3 = False +PY32 = False + +if sys.version_info >= (2, 0) and sys.version_info < (3, 0): + PY2 = True + +if sys.version_info >= (2, 5) and sys.version_info < (2, 6): + PY25 = True + +if sys.version_info >= (2, 6) and sys.version_info < (2, 7): + PY26 = True + +if sys.version_info >= (2, 7) and sys.version_info < (2, 8): + PY27 = True + +if sys.version_info >= (3, 0): + PY3 = True + +if sys.version_info >= (3, 2) and sys.version_info < (3, 3): + PY32 = True + +if PY2_pre_279 or PY3_pre_32: + from backports.ssl_match_hostname import match_hostname, CertificateError # NOQA +else: + # ssl module in Python >= 3.2 includes match hostname function + from ssl import match_hostname, CertificateError # NOQA + +if PY3: + import http.client as httplib + from io import StringIO + import urllib + import urllib as urllib2 + # pylint: disable=no-name-in-module + import urllib.parse as urlparse + import xmlrpc.client as xmlrpclib + + from urllib.parse import quote as urlquote + from urllib.parse import unquote as urlunquote + from urllib.parse import urlencode as urlencode + from os.path import relpath + + from imp import reload + + from builtins import bytes + from builtins import next + + parse_qs = urlparse.parse_qs + parse_qsl = urlparse.parse_qsl + + basestring = str + + def method_type(callable, instance, klass): + return types.MethodType(callable, instance or klass()) + + def b(s): + if isinstance(s, str): + return s.encode('utf-8') + elif isinstance(s, bytes): + return s + elif isinstance(s, int): + return bytes([s]) + else: + raise TypeError("Invalid argument %r for b()" % (s,)) + + def ensure_string(s): + if isinstance(s, str): + return s + elif isinstance(s, bytes): + return s.decode('utf-8') + else: + raise TypeError("Invalid argument %r for ensure_string()" % (s,)) + + def byte(n): + # assume n is a Latin-1 string of length 1 + return ord(n) + + _real_unicode = str + u = str + + def bchr(s): + """Take an integer and make a 1-character byte string.""" + return bytes([s]) + + def dictvalues(d): + return list(d.values()) + + def tostring(node): + return ET.tostring(node, encoding='unicode') + + def hexadigits(s): + # s needs to be a byte string. + return [format(x, "x") for x in s] + +else: + import httplib # NOQA + from StringIO import StringIO # NOQA + import urllib # NOQA + import urllib2 # NOQA + import urlparse # NOQA + import xmlrpclib # NOQA + from urllib import quote as _urlquote # NOQA + from urllib import unquote as urlunquote # NOQA + from urllib import urlencode as urlencode # NOQA + + from __builtin__ import reload # NOQA + + if PY25: + import cgi + + parse_qs = cgi.parse_qs + parse_qsl = cgi.parse_qsl + else: + parse_qs = urlparse.parse_qs + parse_qsl = urlparse.parse_qsl + + if not PY25: + from os.path import relpath # NOQA + + # Save the real value of unicode because urlquote needs it to tell the + # difference between a unicode string and a byte string. + _real_unicode = unicode + basestring = unicode = str + + method_type = types.MethodType + + b = bytes = ensure_string = str + + def byte(n): + return n + + u = unicode + + def bchr(s): + """Take an integer and make a 1-character byte string.""" + return chr(s) + + def next(i): + return i.next() + + def dictvalues(d): + return d.values() + + tostring = ET.tostring + + def urlquote(s, safe='/'): + if isinstance(s, _real_unicode): + # Pretend to be py3 by encoding the URI automatically. + s = s.encode('utf8') + return _urlquote(s, safe) + + def hexadigits(s): + # s needs to be a string. + return [x.encode("hex") for x in s] + +if PY25: + import posixpath + + # Taken from http://jimmyg.org/work/code/barenecessities/index.html + # (MIT license) + # pylint: disable=function-redefined + def relpath(path, start=posixpath.curdir): # NOQA + """Return a relative version of a path""" + if not path: + raise ValueError("no path specified") + start_list = posixpath.abspath(start).split(posixpath.sep) + path_list = posixpath.abspath(path).split(posixpath.sep) + # Work out how much of the filepath is shared by start and path. + i = len(posixpath.commonprefix([start_list, path_list])) + rel_list = [posixpath.pardir] * (len(start_list) - i) + path_list[i:] + if not rel_list: + return posixpath.curdir + return posixpath.join(*rel_list) + +if PY27 or PY3: + unittest2_required = False +else: + unittest2_required = True diff --git a/contrib/tajo-python/src/tajo/queries.py b/contrib/tajo-python/src/tajo/queries.py new file mode 100644 index 0000000000..9e8908b0f8 --- /dev/null +++ b/contrib/tajo-python/src/tajo/queries.py @@ -0,0 +1,61 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from tajo.error import InvalidStatusError +from tajo.base import TajoRequest, TajoObject +from tajo.py3 import httplib, PY3 + +try: + import simplejson as json +except ImportError: + import json + +class TajoQueryInfo(TajoObject): + def __init__(self, objs): + self.objs = objs + + def __repr__(self): + return "(%s:%s)"%(self.objs["queryIdStr"], self.objs["sql"]) + + @staticmethod + def create(headers, content): + if PY3: + content = content.decode('utf-8') + + queries = [] + objs = json.loads(content)["queries"] + for obj in objs: + queries.append(TajoQueryInfo(obj)) + + return queries + +class TajoQueriesRequest(TajoRequest): + object_cls = TajoQueryInfo + ok_status = [httplib.OK] + + def __init__(self, database_name): + self.database_name = database_name + + def uri(self): + return "queries" + + def headers(self): + return None + + def params(self): + return None + + def cls(self): + return self.object_cls diff --git a/contrib/tajo-python/src/tajo/query.py b/contrib/tajo-python/src/tajo/query.py new file mode 100644 index 0000000000..d184ff06bd --- /dev/null +++ b/contrib/tajo-python/src/tajo/query.py @@ -0,0 +1,78 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from tajo.error import InvalidStatusError +from tajo.base import TajoPostRequest, TajoObject +from tajo.querystate import QueryState +from tajo.queryid import QueryId +from tajo.py3 import httplib, PY3 + +try: + import simplejson as json +except ImportError: + import json + +class TajoQuery(TajoObject): + def __init__(self, headers, contents=None): + if PY3: + contents = contents.decode('utf-8') + + self.completed = False + self.objs = json.loads(contents) + if "uri" in self.objs: + self.url = self.objs["uri"] + self.query_id = self.get_parse_query_id(self.url) + else: + self.query_id = QueryId.NULL_QUERY_ID + self.completed = True + self.status = QueryState.QUERY_SUCCEEDED + + def get_query_id(self): + return self.query_id + + def get_parse_query_id(self, url): + parts = url.split('/') + return parts[-1] + + def __repr__(self): + return str(self.uri) + + @staticmethod + def create(headers, contents): + return TajoQuery(headers, contents) + + +class TajoQueryRequest(TajoPostRequest): + object_cls = TajoQuery + ok_status = [httplib.CREATED, httplib.OK] + + def __init__(self, query): + self.query = query + + def uri(self): + return "queries" + + def headers(self): + return None + + def params(self): + payload = { + 'query': self.query + } + + return payload + + def cls(self): + return self.object_cls diff --git a/contrib/tajo-python/src/tajo/queryid.py b/contrib/tajo-python/src/tajo/queryid.py new file mode 100644 index 0000000000..8e89fcdc8f --- /dev/null +++ b/contrib/tajo-python/src/tajo/queryid.py @@ -0,0 +1,17 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +class QueryId: + NULL_QUERY_ID = "q_0000000000000_0000" diff --git a/contrib/tajo-python/src/tajo/querystate.py b/contrib/tajo-python/src/tajo/querystate.py new file mode 100644 index 0000000000..3ee4a874f5 --- /dev/null +++ b/contrib/tajo-python/src/tajo/querystate.py @@ -0,0 +1,27 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +class QueryState: + QUERY_MASTER_INIT = "QUERY_MASTER_INIT" + QUERY_MASTER_LAUNCHED = "QUERY_MASTER_LAUNCHED" + QUERY_NEW = "QUERY_NEW" + QUERY_INIT = "QUERY_INIT" + QUERY_RUNNING = "QUERY_RUNNING" + QUERY_SUCCEEDED = "QUERY_SUCCEEDED" + QUERY_FAILED = "QUERY_FAILED" + QUERY_KILLED = "QUERY_KILLED" + QUERY_KILL_WAIT = "QUERY_KILL_WAIT" + QUERY_ERROR = "QUERY_ERROR" + QUERY_NOT_ASSIGNED = "QUERY_NOT_ASSIGNED" diff --git a/contrib/tajo-python/src/tajo/querystatus.py b/contrib/tajo-python/src/tajo/querystatus.py new file mode 100644 index 0000000000..e7a7efb22b --- /dev/null +++ b/contrib/tajo-python/src/tajo/querystatus.py @@ -0,0 +1,58 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from tajo.error import InvalidStatusError +from tajo.base import TajoRequest, TajoObject +from tajo.py3 import httplib, PY3 + +try: + import simplejson as json +except ImportError: + import json + +class TajoQueryStatus(TajoObject): + def __init__(self, state): + self.state = state + + def __repr__(self): + return self.state + + @staticmethod + def create(headers, content): + if PY3: + content = content.decode('utf-8') + + return TajoQueryStatus(json.loads(content)["queryState"]) + + +class TajoQueryStatusRequest(TajoRequest): + object_cls = TajoQueryStatus + ok_status = [httplib.OK] + + def __init__(self, query_id, base): + self.url = query_id.url + self.base = base + + def uri(self): + return self.url[len(self.base):] + + def headers(self): + return None + + def params(self): + return None + + def cls(self): + return self.object_cls diff --git a/contrib/tajo-python/src/tajo/resultset.py b/contrib/tajo-python/src/tajo/resultset.py new file mode 100644 index 0000000000..eae7bd9d93 --- /dev/null +++ b/contrib/tajo-python/src/tajo/resultset.py @@ -0,0 +1,111 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from tajo.error import InvalidStatusError +from tajo.base import TajoRequest, TajoObject +from tajo.rowdecoder import RowDecoder +from tajo.resultsetbase import ResultSetBase +from tajo.py3 import httplib +import struct + +try: + import simplejson as json +except ImportError: + import json + +class TajoMemoryResultSet(ResultSetBase): + def __init__(self, resultset_info, eos, start_offset, count, content): + super(TajoMemoryResultSet, self).__init__() + self.schema = None + self.decoder = None + self.total_size = 0 + + if resultset_info is not None: + self.schema = resultset_info.schema() + self.decoder = RowDecoder(self.schema) + + if content is not None: + self.total_size = len(content) + + self.total_row = count + self.start_offset = start_offset + self.data = content + self.eos = eos + + self.cur = None + self.data_offset = 0 + + def next_tuple(self): + if self.data_offset < self.total_size: + dsize = struct.unpack('>I', self.data[self.data_offset:self.data_offset+4])[0] + self.data_offset += 4 + tuple_data = self.data[self.data_offset:self.data_offset+dsize] + self.data_offset += dsize + self.cur = self.decoder.toTuple(tuple_data) + return self.cur + + return None + + @staticmethod + def create(resultset_info, headers, content): + offset = 0 + count = 0 + eos = False + + if "x-tajo-offset" in headers: + offset = int(headers["x-tajo-offset"]) + + if "x-tajo-count" in headers: + count = int(headers["x-tajo-count"]) + + if "x-tajo-eos" in headers: + if headers["x-tajo-eos"] == "true": + eos = True + + if offset < 0: + offset = 0 + + return TajoMemoryResultSet(resultset_info, eos, offset, count, content) + + +class TajoMemoryResultSetRequest(TajoRequest): + object_cls = TajoMemoryResultSet + ok_status = [httplib.OK] + + def __init__(self, info, base, count): + self.info = info + self.url = str(info) + self.base = base + self.count = count + + def uri(self): + body = self.url[len(self.base):] + url = "%s?count=%s"%(body, self.count) + + return url + + def headers(self): + return {'Content-Type': 'application/octet-stream'} + + def params(self): + return None + + def cls(self): + return self.object_cls + + def request(self, conn): + headers, contents = conn._request(self.method(), self.uri(), self.params()) + self.check_status(headers, contents) + return self.object_cls.create(self.info, headers, contents) diff --git a/contrib/tajo-python/src/tajo/resultsetbase.py b/contrib/tajo-python/src/tajo/resultsetbase.py new file mode 100644 index 0000000000..50ef89f2e3 --- /dev/null +++ b/contrib/tajo-python/src/tajo/resultsetbase.py @@ -0,0 +1,40 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +class ResultSetBase(object): + def __init__(self): + self.cur_row = 0 + self.cur = None + self.schema = None + self.total_row = 0 + + def current_tuple(self): + return self.cur + + def next(self): + if self.total_row <= 0: + return False + + self.cur = self.next_tuple() + self.cur_row += 1 + if self.cur is not None: + return True + + return False + + def next_tuple(self): + raise Exception("Not Implemented") + + diff --git a/contrib/tajo-python/src/tajo/resultsetinfo.py b/contrib/tajo-python/src/tajo/resultsetinfo.py new file mode 100644 index 0000000000..012794f782 --- /dev/null +++ b/contrib/tajo-python/src/tajo/resultsetinfo.py @@ -0,0 +1,75 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from tajo.error import InvalidStatusError +from tajo.base import TajoRequest, TajoObject +from tajo.py3 import httplib, PY3 + +try: + import simplejson as json +except ImportError: + import json + +class TajoResultSetInfo(TajoObject): + def __init__(self, objs, schema=None): + if objs is not None: + self.objs = objs + self.result_link = self.objs["resultset"]["link"] + + if objs is None and schema is not None: + self.objs = { "schema": schema } + self.result_link = "" + + def schema(self): + return self.objs["schema"] + + def link(self): + return self.result_link + + def __repr__(self): + return self.result_link + + @staticmethod + def create(headers, content): + if PY3: + content = content.decode('utf-8') + + objs = json.loads(content) + if objs['resultCode'] != "OK": + raise InvalidStatusError(int(headers['status'])) + + return TajoResultSetInfo(objs) + + +class TajoResultSetInfoRequest(TajoRequest): + object_cls = TajoResultSetInfo + ok_status = [httplib.OK] + + def __init__(self, query_id, base): + self.url = query_id.url + self.base = base + + def uri(self): + url = "%s/result"%(self.url[len(self.base):]) + return url + + def headers(self): + return None + + def params(self): + return None + + def cls(self): + return self.object_cls diff --git a/contrib/tajo-python/src/tajo/rowdecoder.py b/contrib/tajo-python/src/tajo/rowdecoder.py new file mode 100644 index 0000000000..f1277b96d3 --- /dev/null +++ b/contrib/tajo-python/src/tajo/rowdecoder.py @@ -0,0 +1,84 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import io +import struct +import math +from tajo.schema import Schema +from tajo.datatypes import TajoDataTypes as ttype + +class RowDecoder: + def __init__(self, schema): + self.schema = Schema(schema) + self.headerSize = int(math.ceil(float(len(self.schema.columns)) / 8)) + + def toTuples(self, serializedTuples): + results = [] + for serializedTuple in serializedTuples: + results.append(self.toTuple(serializedTuple)) + + return tuple(results) + + def toTuple(self, serializedTuple): + size = len(self.schema.columns) + nullFlags = serializedTuple[:self.headerSize] + bb = io.BytesIO(serializedTuple[self.headerSize:]) + results = [] + + for i in range(size): + column = self.schema.columns[i] + results.append(self.convert(0, column, bb)) + + return tuple(results) + + def convert(self, isNull, column, bb): + ftype = column.datatype + flen = column.length + + if (isNull == 1): + return "NULL" + + if ftype == ttype.INT1: + v = bb.read(1) + return struct.unpack("b", v)[0] + + if ftype == ttype.INT2: + v = bb.read(2) + return struct.unpack(">h", v)[0] + + if ftype == ttype.INT4 or ftype == ttype.DATE: + v = bb.read(4) + return struct.unpack(">i", v)[0] + + if ftype == ttype.INT8 or ftype == ttype.TIME or ftype == ttype.TIMESTAMP: + v = bb.read(8) + return struct.unpack(">q", v)[0] + + if ftype == ttype.FLOAT4: + v = bb.read(4) + return struct.unpack(">f", v)[0] + + if ftype == ttype.FLOAT8: + v = bb.read(8) + return struct.unpack(">d", v)[0] + + if ftype == ttype.CHAR: + return bb.read(flen) + + if ftype == ttype.TEXT or ftype == ttype.BLOB: + l = bb.read(4) + l2 = struct.unpack(">i", l)[0] + v = bb.read(l2) + return v diff --git a/contrib/tajo-python/src/tajo/schema.py b/contrib/tajo-python/src/tajo/schema.py new file mode 100644 index 0000000000..57f40e1412 --- /dev/null +++ b/contrib/tajo-python/src/tajo/schema.py @@ -0,0 +1,45 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from tajo.column import Column + +class Schema: + def __init__(self, objs): + self.columns = [] + nmap = {} + imap = {} + for name in objs['fieldsByQualifiedName']: + imap[name] = objs['fieldsByQualifiedName'][name] + + for column in objs["fields"]: + datatype = column["typeDesc"]["dataType"] + name = column["name"] + + clen = 1 + try: + clen = datatype["len"] + except: + pass + + new_column = Column(name, datatype["type"], clen) + idx = imap[name] + nmap[idx] = new_column + + size = len(nmap) + for idx in range(size): + self.columns.append(nmap[idx]) + + def column(self, idx): + return self.columns[idx] diff --git a/contrib/tajo-python/src/tajo/session.py b/contrib/tajo-python/src/tajo/session.py new file mode 100644 index 0000000000..729911f8ab --- /dev/null +++ b/contrib/tajo-python/src/tajo/session.py @@ -0,0 +1,70 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from tajo.error import InvalidStatusError +from tajo.base import TajoPostRequest, TajoObject +from tajo.py3 import httplib, PY3 + +try: + import simplejson as json +except ImportError: + import json + +class TajoSession(TajoObject): + def __init__(self, session_id): + self.session_id = session_id + + def __repr__(self): + return str(self.session_id) + + @staticmethod + def create(headers, content): + if PY3: + content = content.decode("utf-8") + + objs = json.loads(content) + if objs["resultCode"] != "OK": + raise InvalidStatusError("parse body failed!") + + if "id" not in objs: + raise InvalidStatusError("parse body failed!") + + return TajoSession(objs["id"]) + + +class TajoSessionRequest(TajoPostRequest): + object_cls = TajoSession + ok_status = [httplib.CREATED] + + def __init__(self, username, database_name): + self.username = username + self.database_name = database_name + + def uri(self): + return "sessions" + + def headers(self): + return None + + def params(self): + payload = { + 'userName': self.username, + 'databaseName': self.database_name + } + + return payload + + def cls(self): + return self.object_cls diff --git a/contrib/tajo-python/src/tajo/tables.py b/contrib/tajo-python/src/tajo/tables.py new file mode 100644 index 0000000000..8a603cd8dc --- /dev/null +++ b/contrib/tajo-python/src/tajo/tables.py @@ -0,0 +1,101 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from tajo.error import InvalidStatusError +from tajo.base import TajoRequest, TajoObject, TajoPostRequest, TajoDeleteRequest +from tajo.py3 import httplib, PY3 + +try: + import simplejson as json +except ImportError: + import json + +class TajoTable(TajoObject): + def __init__(self, name): + self.name = name + + def __repr__(self): + return "%s"%(self.name) + + @staticmethod + def create(headers, content): + if PY3: + content = content.decode('utf-8') + + data = json.loads(content) + return data + + +class TajoTableInfo(TajoObject): + def __init__(self, name): + self.name = name + + def __repr__(self): + return "%s"%(self.name) + + +class TajoTablesInfo(TajoObject): + @staticmethod + def create(headers, content): + if PY3: + content = content.decode('utf-8') + + data = json.loads(content) + tables = [] + for table in data['tables']: + tables.append(TajoTableInfo(table)) + + return tables + + +class TajoTablesRequest(TajoRequest): + object_cls = TajoTablesInfo + ok_status = [httplib.OK] + + def __init__(self, database_name): + self.database_name = database_name + + def uri(self): + return "databases/%s/tables"%(self.database_name) + + def headers(self): + return None + + def params(self): + return None + + def cls(self): + return self.object_cls + + +class TajoTableRequest(TajoRequest): + object_cls = TajoTable + ok_status = [httplib.OK] + + def __init__(self, database_name, table_name): + self.database_name = database_name + self.table_name = table_name + + def uri(self): + return "databases/%s/tables/%s"%(self.database_name, self.table_name) + + def headers(self): + return None + + def params(self): + return None + + def cls(self): + return self.object_cls diff --git a/contrib/tajo-python/src/tajo_cluster.py b/contrib/tajo-python/src/tajo_cluster.py new file mode 100644 index 0000000000..f648512a2d --- /dev/null +++ b/contrib/tajo-python/src/tajo_cluster.py @@ -0,0 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from tajo.client import TajoClient + +client = TajoClient("http://127.0.0.1:26880/rest", username='tajo') +cluster_info = client.cluster_info() +print(cluster_info) diff --git a/contrib/tajo-python/src/tajo_databases.py b/contrib/tajo-python/src/tajo_databases.py new file mode 100644 index 0000000000..e10da0c48c --- /dev/null +++ b/contrib/tajo-python/src/tajo_databases.py @@ -0,0 +1,29 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from tajo.client import TajoClient + +client = TajoClient("http://127.0.0.1:26880/rest", username='tajo') +client.create_database("test123") +databases = client.databases() + +for database_name in databases: + try: + ret = client.database_info(database_name) + print(ret) + except: + pass + +client.delete_database("test123") diff --git a/contrib/tajo-python/src/tajo_example.py b/contrib/tajo-python/src/tajo_example.py new file mode 100644 index 0000000000..465fdc98ec --- /dev/null +++ b/contrib/tajo-python/src/tajo_example.py @@ -0,0 +1,32 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from tajo.client import TajoClient + +client = TajoClient("http://127.0.0.1:26880/rest", username='tajo') +client.execute_query_wait_result('create table "Test2" (col1 int)') +client.execute_query_wait_result('insert into "Test2" select 1') +client.execute_query_wait_result('insert into "Test2" select 2') +client.execute_query_wait_result('insert into "Test2" select 3') + +resultset = client.execute_query_wait_result('select * from "Test2"') +while True: + t = resultset.next_tuple() + if t is None: + break + + print(t) + +client.execute_query_wait_result('drop table "Test2" purge') diff --git a/contrib/tajo-python/src/tajo_functions.py b/contrib/tajo-python/src/tajo_functions.py new file mode 100644 index 0000000000..3adb416293 --- /dev/null +++ b/contrib/tajo-python/src/tajo_functions.py @@ -0,0 +1,21 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from tajo.client import TajoClient + +client = TajoClient("http://127.0.0.1:26880/rest", username='tajo') +functions = client.functions() +for f in functions: + print(f) diff --git a/contrib/tajo-python/src/tajo_queries.py b/contrib/tajo-python/src/tajo_queries.py new file mode 100644 index 0000000000..d0ddbd904d --- /dev/null +++ b/contrib/tajo-python/src/tajo_queries.py @@ -0,0 +1,21 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from tajo.client import TajoClient + +client = TajoClient("http://127.0.0.1:26880/rest", username='tajo') +queries = client.queries() +for query in queries: + print(query) diff --git a/contrib/tajo-python/src/tajo_tables.py b/contrib/tajo-python/src/tajo_tables.py new file mode 100644 index 0000000000..84ce251bd1 --- /dev/null +++ b/contrib/tajo-python/src/tajo_tables.py @@ -0,0 +1,34 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from tajo.client import TajoClient + +client = TajoClient("http://127.0.0.1:26880/rest", username='tajo') + +client.execute_query_wait_result('create table Test1 (col1 int)') +client.execute_query_wait_result('create table Test2 (col1 int)') +client.execute_query_wait_result('create table Test3 (col1 int)') +client.execute_query_wait_result('create table Test4 (col1 int)') + +tables = client.tables("default") +print(tables) +for table in tables: + t = client.table("default", table.name) + print(t) + +client.execute_query_wait_result('drop table Test1 purge') +client.execute_query_wait_result('drop table Test2 purge') +client.execute_query_wait_result('drop table Test3 purge') +client.execute_query_wait_result('drop table Test4 purge')