timeplus.stream
stream
This module defines stream class :copyright: (c) 2022 by Timeplus :license: Apache2, see LICENSE for more details.
View Source
""" stream This module defines stream class :copyright: (c) 2022 by Timeplus :license: Apache2, see LICENSE for more details. """ import requests import json from datetime import datetime from timeplus.base import Base from timeplus.resource import ResourceBase from timeplus.error import TimeplusAPIError from timeplus.type import Type class DateTimeEncoder(json.JSONEncoder): """ DateTimeEncoder class defines how to encode datetime object for json """ def default(self, o): if isinstance(o, datetime): return o.isoformat() return json.JSONEncoder.default(self, o) class StreamColumn(Base): """ StreamColumn class defines columne of a stream """ def __init__(self): Base.__init__(self) def name(self, *args): return self.prop("name", *args) def type(self, *args): if len(args) == 0: return Type(self._get("type")) elif len(args) >= 1: if isinstance(args[0], Type): if args[0] == Type.Decimal: # TODO : need validate the args decimal_type = f"{args[0].value}({args[1]}, {args[2]})" return self._set("type", decimal_type) elif args[0] == Type.Array and isinstance(args[1], Type): array_type = f"{args[0].value}({args[1].value})" return self._set("type", array_type) elif ( args[0] == Type.Map and isinstance(args[1], Type) and isinstance(args[2], Type) ): map_type = f"{args[0].value}({args[1].value}, {args[2].value})" return self._set("type", map_type) elif args[0] == Type.Tuple: tuple_values = ",".join([a.value for a in args[1:]]) map_type = f"{args[0].value}({tuple_values})" return self._set("type", map_type) else: return self._set("type", args[0].value) else: return self._set("type", args[0]) def default(self, *args): return self.prop("default", *args) def compression_codec(self, *args): return self.prop("compression_codec", *args) def nullable(self, *args): return self.prop("nullable", *args) def skipping_index_expression(self, *args): return self.prop("skipping_index_expression", *args) def ttl_expression(self, *args): return self.prop("ttl_expression", *args) class Stream(ResourceBase): """ Stream class defines stream object """ _resource_name = "streams" def __init__(self, env=None): ResourceBase.__init__(self, env) self.prop("columns", []) @classmethod def build(cls, val, env=None): obj = cls(env=env) obj._data = val return obj # the list api is not implemented, has to manually implement it here def get(self): streams = Stream.list() for s in streams: if s.name() == self.name(): return s def delete(self): url = f"{self._base_url}/{self._resource_name}/{self.name()}" self._logger.debug("delete {}", url) try: r = requests.delete( url, headers=self._headers, timeout=self._env.http_timeout(), ) if r.status_code < 200 or r.status_code > 299: err_msg = f"failed to delete {self._resource_name} due to {r.text}" raise TimeplusAPIError("delete", r.status_code, err_msg) else: self._logger.debug(f"delete {self._resource_name} success") return self except Exception as e: self._logger.error(f"failed to delete {e}") raise e def insert(self, data, headers=None): url = f"{self._base_url}/{self._resource_name}/{self.name()}/ingest" self._logger.debug("post {}", url) insert_headers = self.columnNames() if headers is not None: insert_headers = headers insertRequest = { "columns": insert_headers, "data": data, } self._logger.debug(f"insert {insertRequest}") try: self._logger.debug( f"insert {json.dumps(insertRequest, cls=DateTimeEncoder)}" ) r = requests.post( url, data=json.dumps(insertRequest, cls=DateTimeEncoder), headers=self._headers, timeout=self._env.http_timeout(), ) if r.status_code < 200 or r.status_code > 299: err_msg = f"failed to insert into {self._resource_name} due to {r.text}" raise TimeplusAPIError("post", r.status_code, err_msg) else: self._logger.debug("insert success") return self except Exception as e: self._logger.error(f"failed to insert {e}") raise e def name(self, *args): return self.prop("name", *args) def event_time_column(self, *args): return self.prop("event_time_column", *args) def order_by_expression(self, *args): return self.prop("order_by_expression", *args) def order_by_granularity(self, *args): return self.prop("order_by_granularity", *args) def partition_by_granularity(self, *args): return self.prop("partition_by_granularity", *args) def replication_factor(self, *args): return self.prop("replication_factor", *args) def shards(self, *args): return self.prop("shards", *args) def ttl_expression(self, *args): return self.prop("ttl_expression", *args) # note, no way to remove/rename col for now def column(self, col): col_prop = self.prop("columns") col_prop.append(col.data()) self.prop("columns", col_prop) return self def columnNames(self): col_prop = self.prop("columns") return [x["name"] for x in col_prop if not x["name"].startswith("_tp")]
View Source
class DateTimeEncoder(json.JSONEncoder): """ DateTimeEncoder class defines how to encode datetime object for json """ def default(self, o): if isinstance(o, datetime): return o.isoformat() return json.JSONEncoder.default(self, o)
DateTimeEncoder class defines how to encode datetime object for json
View Source
def default(self, o): if isinstance(o, datetime): return o.isoformat() return json.JSONEncoder.default(self, o)
Implement this method in a subclass such that it returns
a serializable object for o
, or calls the base implementation
(to raise a TypeError
).
For example, to support arbitrary iterators, you could implement default like this::
def default(self, o):
try:
iterable = iter(o)
except TypeError:
pass
else:
return list(iterable)
# Let the base class default method raise the TypeError
return JSONEncoder.default(self, o)
Inherited Members
- json.encoder.JSONEncoder
- JSONEncoder
- item_separator
- key_separator
- encode
- iterencode
View Source
class StreamColumn(Base): """ StreamColumn class defines columne of a stream """ def __init__(self): Base.__init__(self) def name(self, *args): return self.prop("name", *args) def type(self, *args): if len(args) == 0: return Type(self._get("type")) elif len(args) >= 1: if isinstance(args[0], Type): if args[0] == Type.Decimal: # TODO : need validate the args decimal_type = f"{args[0].value}({args[1]}, {args[2]})" return self._set("type", decimal_type) elif args[0] == Type.Array and isinstance(args[1], Type): array_type = f"{args[0].value}({args[1].value})" return self._set("type", array_type) elif ( args[0] == Type.Map and isinstance(args[1], Type) and isinstance(args[2], Type) ): map_type = f"{args[0].value}({args[1].value}, {args[2].value})" return self._set("type", map_type) elif args[0] == Type.Tuple: tuple_values = ",".join([a.value for a in args[1:]]) map_type = f"{args[0].value}({tuple_values})" return self._set("type", map_type) else: return self._set("type", args[0].value) else: return self._set("type", args[0]) def default(self, *args): return self.prop("default", *args) def compression_codec(self, *args): return self.prop("compression_codec", *args) def nullable(self, *args): return self.prop("nullable", *args) def skipping_index_expression(self, *args): return self.prop("skipping_index_expression", *args) def ttl_expression(self, *args): return self.prop("ttl_expression", *args)
StreamColumn class defines columne of a stream
View Source
def __init__(self): Base.__init__(self)
View Source
def name(self, *args): return self.prop("name", *args)
View Source
def type(self, *args): if len(args) == 0: return Type(self._get("type")) elif len(args) >= 1: if isinstance(args[0], Type): if args[0] == Type.Decimal: # TODO : need validate the args decimal_type = f"{args[0].value}({args[1]}, {args[2]})" return self._set("type", decimal_type) elif args[0] == Type.Array and isinstance(args[1], Type): array_type = f"{args[0].value}({args[1].value})" return self._set("type", array_type) elif ( args[0] == Type.Map and isinstance(args[1], Type) and isinstance(args[2], Type) ): map_type = f"{args[0].value}({args[1].value}, {args[2].value})" return self._set("type", map_type) elif args[0] == Type.Tuple: tuple_values = ",".join([a.value for a in args[1:]]) map_type = f"{args[0].value}({tuple_values})" return self._set("type", map_type) else: return self._set("type", args[0].value) else: return self._set("type", args[0])
View Source
def default(self, *args): return self.prop("default", *args)
View Source
def compression_codec(self, *args): return self.prop("compression_codec", *args)
View Source
def nullable(self, *args): return self.prop("nullable", *args)
View Source
def skipping_index_expression(self, *args): return self.prop("skipping_index_expression", *args)
View Source
def ttl_expression(self, *args): return self.prop("ttl_expression", *args)
Inherited Members
View Source
class Stream(ResourceBase): """ Stream class defines stream object """ _resource_name = "streams" def __init__(self, env=None): ResourceBase.__init__(self, env) self.prop("columns", []) @classmethod def build(cls, val, env=None): obj = cls(env=env) obj._data = val return obj # the list api is not implemented, has to manually implement it here def get(self): streams = Stream.list() for s in streams: if s.name() == self.name(): return s def delete(self): url = f"{self._base_url}/{self._resource_name}/{self.name()}" self._logger.debug("delete {}", url) try: r = requests.delete( url, headers=self._headers, timeout=self._env.http_timeout(), ) if r.status_code < 200 or r.status_code > 299: err_msg = f"failed to delete {self._resource_name} due to {r.text}" raise TimeplusAPIError("delete", r.status_code, err_msg) else: self._logger.debug(f"delete {self._resource_name} success") return self except Exception as e: self._logger.error(f"failed to delete {e}") raise e def insert(self, data, headers=None): url = f"{self._base_url}/{self._resource_name}/{self.name()}/ingest" self._logger.debug("post {}", url) insert_headers = self.columnNames() if headers is not None: insert_headers = headers insertRequest = { "columns": insert_headers, "data": data, } self._logger.debug(f"insert {insertRequest}") try: self._logger.debug( f"insert {json.dumps(insertRequest, cls=DateTimeEncoder)}" ) r = requests.post( url, data=json.dumps(insertRequest, cls=DateTimeEncoder), headers=self._headers, timeout=self._env.http_timeout(), ) if r.status_code < 200 or r.status_code > 299: err_msg = f"failed to insert into {self._resource_name} due to {r.text}" raise TimeplusAPIError("post", r.status_code, err_msg) else: self._logger.debug("insert success") return self except Exception as e: self._logger.error(f"failed to insert {e}") raise e def name(self, *args): return self.prop("name", *args) def event_time_column(self, *args): return self.prop("event_time_column", *args) def order_by_expression(self, *args): return self.prop("order_by_expression", *args) def order_by_granularity(self, *args): return self.prop("order_by_granularity", *args) def partition_by_granularity(self, *args): return self.prop("partition_by_granularity", *args) def replication_factor(self, *args): return self.prop("replication_factor", *args) def shards(self, *args): return self.prop("shards", *args) def ttl_expression(self, *args): return self.prop("ttl_expression", *args) # note, no way to remove/rename col for now def column(self, col): col_prop = self.prop("columns") col_prop.append(col.data()) self.prop("columns", col_prop) return self def columnNames(self): col_prop = self.prop("columns") return [x["name"] for x in col_prop if not x["name"].startswith("_tp")]
Stream class defines stream object
View Source
def __init__(self, env=None): ResourceBase.__init__(self, env) self.prop("columns", [])
View Source
@classmethod def build(cls, val, env=None): obj = cls(env=env) obj._data = val return obj
View Source
def get(self): streams = Stream.list() for s in streams: if s.name() == self.name(): return s
View Source
def delete(self): url = f"{self._base_url}/{self._resource_name}/{self.name()}" self._logger.debug("delete {}", url) try: r = requests.delete( url, headers=self._headers, timeout=self._env.http_timeout(), ) if r.status_code < 200 or r.status_code > 299: err_msg = f"failed to delete {self._resource_name} due to {r.text}" raise TimeplusAPIError("delete", r.status_code, err_msg) else: self._logger.debug(f"delete {self._resource_name} success") return self except Exception as e: self._logger.error(f"failed to delete {e}") raise e
View Source
def insert(self, data, headers=None): url = f"{self._base_url}/{self._resource_name}/{self.name()}/ingest" self._logger.debug("post {}", url) insert_headers = self.columnNames() if headers is not None: insert_headers = headers insertRequest = { "columns": insert_headers, "data": data, } self._logger.debug(f"insert {insertRequest}") try: self._logger.debug( f"insert {json.dumps(insertRequest, cls=DateTimeEncoder)}" ) r = requests.post( url, data=json.dumps(insertRequest, cls=DateTimeEncoder), headers=self._headers, timeout=self._env.http_timeout(), ) if r.status_code < 200 or r.status_code > 299: err_msg = f"failed to insert into {self._resource_name} due to {r.text}" raise TimeplusAPIError("post", r.status_code, err_msg) else: self._logger.debug("insert success") return self except Exception as e: self._logger.error(f"failed to insert {e}") raise e
View Source
def name(self, *args): return self.prop("name", *args)
View Source
def event_time_column(self, *args): return self.prop("event_time_column", *args)
View Source
def order_by_expression(self, *args): return self.prop("order_by_expression", *args)
View Source
def order_by_granularity(self, *args): return self.prop("order_by_granularity", *args)
View Source
def partition_by_granularity(self, *args): return self.prop("partition_by_granularity", *args)
View Source
def replication_factor(self, *args): return self.prop("replication_factor", *args)
View Source
def shards(self, *args): return self.prop("shards", *args)
View Source
def ttl_expression(self, *args): return self.prop("ttl_expression", *args)
View Source
def column(self, col): col_prop = self.prop("columns") col_prop.append(col.data()) self.prop("columns", col_prop) return self
View Source
def columnNames(self): col_prop = self.prop("columns") return [x["name"] for x in col_prop if not x["name"].startswith("_tp")]