timeplus.stream
stream
This module defines stream class
:copyright: (c) 2022 by Timeplus
:license: Apache2, see LICENSE for more details.
View Source
0""" 1stream 2 3This module defines stream class 4:copyright: (c) 2022 by Timeplus 5:license: Apache2, see LICENSE for more details. 6""" 7 8import requests 9import json 10from datetime import datetime 11 12from timeplus.base import Base 13from timeplus.resource import ResourceBase 14from timeplus.error import TimeplusAPIError 15 16from timeplus.type import Type 17 18 19class DateTimeEncoder(json.JSONEncoder): 20 """ 21 DateTimeEncoder class defines how to encode datetime object for json 22 """ 23 24 def default(self, o): 25 if isinstance(o, datetime): 26 return o.isoformat() 27 return json.JSONEncoder.default(self, o) 28 29 30class StreamColumn(Base): 31 """ 32 StreamColumn class defines columne of a stream 33 """ 34 35 def __init__(self): 36 Base.__init__(self) 37 38 def name(self, *args): 39 return self.prop("name", *args) 40 41 def type(self, *args): 42 if len(args) == 0: 43 return Type(self._get("type")) 44 elif len(args) >= 1: 45 if isinstance(args[0], Type): 46 if args[0] == Type.Decimal: 47 # TODO : need validate the args 48 decimal_type = f"{args[0].value}({args[1]}, {args[2]})" 49 return self._set("type", decimal_type) 50 elif args[0] == Type.Array and isinstance(args[1], Type): 51 array_type = f"{args[0].value}({args[1].value})" 52 return self._set("type", array_type) 53 elif ( 54 args[0] == Type.Map 55 and isinstance(args[1], Type) 56 and isinstance(args[2], Type) 57 ): 58 map_type = f"{args[0].value}({args[1].value}, {args[2].value})" 59 return self._set("type", map_type) 60 elif args[0] == Type.Tuple: 61 tuple_values = ",".join([a.value for a in args[1:]]) 62 map_type = f"{args[0].value}({tuple_values})" 63 return self._set("type", map_type) 64 else: 65 return self._set("type", args[0].value) 66 67 else: 68 return self._set("type", args[0]) 69 70 def default(self, *args): 71 return self.prop("default", *args) 72 73 def compression_codec(self, *args): 74 return self.prop("compression_codec", *args) 75 76 def nullable(self, *args): 77 return self.prop("nullable", *args) 78 79 def skipping_index_expression(self, *args): 80 return self.prop("skipping_index_expression", *args) 81 82 def ttl_expression(self, *args): 83 return self.prop("ttl_expression", *args) 84 85 86class Stream(ResourceBase): 87 """ 88 Stream class defines stream object 89 """ 90 91 _resource_name = "streams" 92 93 def __init__(self, env=None): 94 ResourceBase.__init__(self, env) 95 self.prop("columns", []) 96 97 @classmethod 98 def build(cls, val, env=None): 99 obj = cls(env=env) 100 obj._data = val 101 return obj 102 103 # the list api is not implemented, has to manually implement it here 104 def get(self): 105 streams = Stream.list() 106 for s in streams: 107 if s.name() == self.name(): 108 return s 109 110 def delete(self): 111 url = f"{self._base_url}/{self._resource_name}/{self.name()}" 112 self._logger.debug("delete {}", url) 113 try: 114 r = requests.delete( 115 url, 116 headers=self._headers, 117 timeout=self._env.http_timeout(), 118 ) 119 if r.status_code < 200 or r.status_code > 299: 120 err_msg = f"failed to delete {self._resource_name} due to {r.text}" 121 raise TimeplusAPIError("delete", r.status_code, err_msg) 122 else: 123 self._logger.debug(f"delete {self._resource_name} success") 124 return self 125 except Exception as e: 126 self._logger.error(f"failed to delete {e}") 127 raise e 128 129 def insert(self, data, headers=None): 130 url = f"{self._base_url}/{self._resource_name}/{self.name()}/ingest" 131 self._logger.debug("post {}", url) 132 133 insert_headers = self.columnNames() 134 if headers is not None: 135 insert_headers = headers 136 137 insertRequest = { 138 "columns": insert_headers, 139 "data": data, 140 } 141 self._logger.debug(f"insert {insertRequest}") 142 143 try: 144 self._logger.debug( 145 f"insert {json.dumps(insertRequest, cls=DateTimeEncoder)}" 146 ) 147 r = requests.post( 148 url, 149 data=json.dumps(insertRequest, cls=DateTimeEncoder), 150 headers=self._headers, 151 timeout=self._env.http_timeout(), 152 ) 153 if r.status_code < 200 or r.status_code > 299: 154 err_msg = f"failed to insert into {self._resource_name} due to {r.text}" 155 raise TimeplusAPIError("post", r.status_code, err_msg) 156 else: 157 self._logger.debug("insert success") 158 return self 159 except Exception as e: 160 self._logger.error(f"failed to insert {e}") 161 raise e 162 163 def name(self, *args): 164 return self.prop("name", *args) 165 166 def event_time_column(self, *args): 167 return self.prop("event_time_column", *args) 168 169 def order_by_expression(self, *args): 170 return self.prop("order_by_expression", *args) 171 172 def order_by_granularity(self, *args): 173 return self.prop("order_by_granularity", *args) 174 175 def partition_by_granularity(self, *args): 176 return self.prop("partition_by_granularity", *args) 177 178 def replication_factor(self, *args): 179 return self.prop("replication_factor", *args) 180 181 def shards(self, *args): 182 return self.prop("shards", *args) 183 184 def ttl_expression(self, *args): 185 return self.prop("ttl_expression", *args) 186 187 # note, no way to remove/rename col for now 188 def column(self, col): 189 col_prop = self.prop("columns") 190 col_prop.append(col.data()) 191 self.prop("columns", col_prop) 192 return self 193 194 def columnNames(self): 195 col_prop = self.prop("columns") 196 return [x["name"] for x in col_prop if not x["name"].startswith("_tp")]
View Source
DateTimeEncoder class defines how to encode datetime object for json
View Source
Implement this method in a subclass such that it returns
a serializable object for o
, or calls the base implementation
(to raise a TypeError
).
For example, to support arbitrary iterators, you could implement default like this::
def default(self, o):
try:
iterable = iter(o)
except TypeError:
pass
else:
return list(iterable)
# Let the base class default method raise the TypeError
return JSONEncoder.default(self, o)
Inherited Members
- json.encoder.JSONEncoder
- JSONEncoder
- item_separator
- key_separator
- encode
- iterencode
View Source
31class StreamColumn(Base): 32 """ 33 StreamColumn class defines columne of a stream 34 """ 35 36 def __init__(self): 37 Base.__init__(self) 38 39 def name(self, *args): 40 return self.prop("name", *args) 41 42 def type(self, *args): 43 if len(args) == 0: 44 return Type(self._get("type")) 45 elif len(args) >= 1: 46 if isinstance(args[0], Type): 47 if args[0] == Type.Decimal: 48 # TODO : need validate the args 49 decimal_type = f"{args[0].value}({args[1]}, {args[2]})" 50 return self._set("type", decimal_type) 51 elif args[0] == Type.Array and isinstance(args[1], Type): 52 array_type = f"{args[0].value}({args[1].value})" 53 return self._set("type", array_type) 54 elif ( 55 args[0] == Type.Map 56 and isinstance(args[1], Type) 57 and isinstance(args[2], Type) 58 ): 59 map_type = f"{args[0].value}({args[1].value}, {args[2].value})" 60 return self._set("type", map_type) 61 elif args[0] == Type.Tuple: 62 tuple_values = ",".join([a.value for a in args[1:]]) 63 map_type = f"{args[0].value}({tuple_values})" 64 return self._set("type", map_type) 65 else: 66 return self._set("type", args[0].value) 67 68 else: 69 return self._set("type", args[0]) 70 71 def default(self, *args): 72 return self.prop("default", *args) 73 74 def compression_codec(self, *args): 75 return self.prop("compression_codec", *args) 76 77 def nullable(self, *args): 78 return self.prop("nullable", *args) 79 80 def skipping_index_expression(self, *args): 81 return self.prop("skipping_index_expression", *args) 82 83 def ttl_expression(self, *args): 84 return self.prop("ttl_expression", *args)
StreamColumn class defines columne of a stream
View Source
42 def type(self, *args): 43 if len(args) == 0: 44 return Type(self._get("type")) 45 elif len(args) >= 1: 46 if isinstance(args[0], Type): 47 if args[0] == Type.Decimal: 48 # TODO : need validate the args 49 decimal_type = f"{args[0].value}({args[1]}, {args[2]})" 50 return self._set("type", decimal_type) 51 elif args[0] == Type.Array and isinstance(args[1], Type): 52 array_type = f"{args[0].value}({args[1].value})" 53 return self._set("type", array_type) 54 elif ( 55 args[0] == Type.Map 56 and isinstance(args[1], Type) 57 and isinstance(args[2], Type) 58 ): 59 map_type = f"{args[0].value}({args[1].value}, {args[2].value})" 60 return self._set("type", map_type) 61 elif args[0] == Type.Tuple: 62 tuple_values = ",".join([a.value for a in args[1:]]) 63 map_type = f"{args[0].value}({tuple_values})" 64 return self._set("type", map_type) 65 else: 66 return self._set("type", args[0].value) 67 68 else: 69 return self._set("type", args[0])
Inherited Members
View Source
87class Stream(ResourceBase): 88 """ 89 Stream class defines stream object 90 """ 91 92 _resource_name = "streams" 93 94 def __init__(self, env=None): 95 ResourceBase.__init__(self, env) 96 self.prop("columns", []) 97 98 @classmethod 99 def build(cls, val, env=None): 100 obj = cls(env=env) 101 obj._data = val 102 return obj 103 104 # the list api is not implemented, has to manually implement it here 105 def get(self): 106 streams = Stream.list() 107 for s in streams: 108 if s.name() == self.name(): 109 return s 110 111 def delete(self): 112 url = f"{self._base_url}/{self._resource_name}/{self.name()}" 113 self._logger.debug("delete {}", url) 114 try: 115 r = requests.delete( 116 url, 117 headers=self._headers, 118 timeout=self._env.http_timeout(), 119 ) 120 if r.status_code < 200 or r.status_code > 299: 121 err_msg = f"failed to delete {self._resource_name} due to {r.text}" 122 raise TimeplusAPIError("delete", r.status_code, err_msg) 123 else: 124 self._logger.debug(f"delete {self._resource_name} success") 125 return self 126 except Exception as e: 127 self._logger.error(f"failed to delete {e}") 128 raise e 129 130 def insert(self, data, headers=None): 131 url = f"{self._base_url}/{self._resource_name}/{self.name()}/ingest" 132 self._logger.debug("post {}", url) 133 134 insert_headers = self.columnNames() 135 if headers is not None: 136 insert_headers = headers 137 138 insertRequest = { 139 "columns": insert_headers, 140 "data": data, 141 } 142 self._logger.debug(f"insert {insertRequest}") 143 144 try: 145 self._logger.debug( 146 f"insert {json.dumps(insertRequest, cls=DateTimeEncoder)}" 147 ) 148 r = requests.post( 149 url, 150 data=json.dumps(insertRequest, cls=DateTimeEncoder), 151 headers=self._headers, 152 timeout=self._env.http_timeout(), 153 ) 154 if r.status_code < 200 or r.status_code > 299: 155 err_msg = f"failed to insert into {self._resource_name} due to {r.text}" 156 raise TimeplusAPIError("post", r.status_code, err_msg) 157 else: 158 self._logger.debug("insert success") 159 return self 160 except Exception as e: 161 self._logger.error(f"failed to insert {e}") 162 raise e 163 164 def name(self, *args): 165 return self.prop("name", *args) 166 167 def event_time_column(self, *args): 168 return self.prop("event_time_column", *args) 169 170 def order_by_expression(self, *args): 171 return self.prop("order_by_expression", *args) 172 173 def order_by_granularity(self, *args): 174 return self.prop("order_by_granularity", *args) 175 176 def partition_by_granularity(self, *args): 177 return self.prop("partition_by_granularity", *args) 178 179 def replication_factor(self, *args): 180 return self.prop("replication_factor", *args) 181 182 def shards(self, *args): 183 return self.prop("shards", *args) 184 185 def ttl_expression(self, *args): 186 return self.prop("ttl_expression", *args) 187 188 # note, no way to remove/rename col for now 189 def column(self, col): 190 col_prop = self.prop("columns") 191 col_prop.append(col.data()) 192 self.prop("columns", col_prop) 193 return self 194 195 def columnNames(self): 196 col_prop = self.prop("columns") 197 return [x["name"] for x in col_prop if not x["name"].startswith("_tp")]
Stream class defines stream object
View Source
111 def delete(self): 112 url = f"{self._base_url}/{self._resource_name}/{self.name()}" 113 self._logger.debug("delete {}", url) 114 try: 115 r = requests.delete( 116 url, 117 headers=self._headers, 118 timeout=self._env.http_timeout(), 119 ) 120 if r.status_code < 200 or r.status_code > 299: 121 err_msg = f"failed to delete {self._resource_name} due to {r.text}" 122 raise TimeplusAPIError("delete", r.status_code, err_msg) 123 else: 124 self._logger.debug(f"delete {self._resource_name} success") 125 return self 126 except Exception as e: 127 self._logger.error(f"failed to delete {e}") 128 raise e
View Source
130 def insert(self, data, headers=None): 131 url = f"{self._base_url}/{self._resource_name}/{self.name()}/ingest" 132 self._logger.debug("post {}", url) 133 134 insert_headers = self.columnNames() 135 if headers is not None: 136 insert_headers = headers 137 138 insertRequest = { 139 "columns": insert_headers, 140 "data": data, 141 } 142 self._logger.debug(f"insert {insertRequest}") 143 144 try: 145 self._logger.debug( 146 f"insert {json.dumps(insertRequest, cls=DateTimeEncoder)}" 147 ) 148 r = requests.post( 149 url, 150 data=json.dumps(insertRequest, cls=DateTimeEncoder), 151 headers=self._headers, 152 timeout=self._env.http_timeout(), 153 ) 154 if r.status_code < 200 or r.status_code > 299: 155 err_msg = f"failed to insert into {self._resource_name} due to {r.text}" 156 raise TimeplusAPIError("post", r.status_code, err_msg) 157 else: 158 self._logger.debug("insert success") 159 return self 160 except Exception as e: 161 self._logger.error(f"failed to insert {e}") 162 raise e