muutils.logger.log_util
1from muutils.jsonlines import jsonl_load_log 2 3 4def get_any_from_stream(stream: list[dict], key: str) -> None: 5 """get the first value of a key from a stream. errors if not found""" 6 for msg in stream: 7 if key in msg: 8 return msg[key] 9 10 raise KeyError(f"key '{key}' not found in stream") 11 12 13def gather_log(file: str) -> dict[str, list[dict]]: 14 """gathers and sorts all streams from a log""" 15 data: list[dict] = jsonl_load_log(file) 16 output: dict[str, list[dict]] = dict() 17 18 for item in data: 19 stream: str = item.get("_stream", None) 20 if stream not in output: 21 output[stream] = list() 22 output[stream].append(item) 23 24 return output 25 26 27def gather_stream( 28 file: str, 29 stream: str, 30) -> list[dict]: 31 """gets all entries from a specific stream in a log file""" 32 data: list[dict] = jsonl_load_log(file) 33 34 output: list[dict] = list() 35 36 for item in data: 37 # select for the stream 38 if ("_stream" in item) and (item["_stream"] == stream): 39 output.append(item) 40 return output 41 42 43def gather_val( 44 file: str, 45 stream: str, 46 keys: tuple[str], 47 allow_skip: bool = True, 48) -> list[list]: 49 """gather specific keys from a specific stream in a log file 50 51 example: 52 if "log.jsonl" has contents: 53 ```jsonl 54 {"a": 1, "b": 2, "c": 3, "_stream": "s1"} 55 {"a": 4, "b": 5, "c": 6, "_stream": "s1"} 56 {"a": 7, "b": 8, "c": 9, "_stream": "s2"} 57 ``` 58 then `gather_val("log.jsonl", "s1", ("a", "b"))` will return 59 ```python 60 [ 61 [1, 2], 62 [4, 5] 63 ] 64 ``` 65 66 """ 67 data: list[dict] = jsonl_load_log(file) 68 69 output: list[list] = list() 70 71 for item in data: 72 # select for the stream 73 if ("_stream" in item) and (item["_stream"] == stream): 74 # select for the keys 75 if all(k in item for k in keys): 76 output.append(list(item[k] for k in keys)) 77 elif not allow_skip: 78 raise ValueError(f"missing keys '{keys = }' in '{item = }'") 79 80 return output
def
get_any_from_stream(stream: list[dict], key: str) -> None:
5def get_any_from_stream(stream: list[dict], key: str) -> None: 6 """get the first value of a key from a stream. errors if not found""" 7 for msg in stream: 8 if key in msg: 9 return msg[key] 10 11 raise KeyError(f"key '{key}' not found in stream")
get the first value of a key from a stream. errors if not found
def
gather_log(file: str) -> dict[str, list[dict]]:
14def gather_log(file: str) -> dict[str, list[dict]]: 15 """gathers and sorts all streams from a log""" 16 data: list[dict] = jsonl_load_log(file) 17 output: dict[str, list[dict]] = dict() 18 19 for item in data: 20 stream: str = item.get("_stream", None) 21 if stream not in output: 22 output[stream] = list() 23 output[stream].append(item) 24 25 return output
gathers and sorts all streams from a log
def
gather_stream(file: str, stream: str) -> list[dict]:
28def gather_stream( 29 file: str, 30 stream: str, 31) -> list[dict]: 32 """gets all entries from a specific stream in a log file""" 33 data: list[dict] = jsonl_load_log(file) 34 35 output: list[dict] = list() 36 37 for item in data: 38 # select for the stream 39 if ("_stream" in item) and (item["_stream"] == stream): 40 output.append(item) 41 return output
gets all entries from a specific stream in a log file
def
gather_val( file: str, stream: str, keys: tuple[str], allow_skip: bool = True) -> list[list]:
44def gather_val( 45 file: str, 46 stream: str, 47 keys: tuple[str], 48 allow_skip: bool = True, 49) -> list[list]: 50 """gather specific keys from a specific stream in a log file 51 52 example: 53 if "log.jsonl" has contents: 54 ```jsonl 55 {"a": 1, "b": 2, "c": 3, "_stream": "s1"} 56 {"a": 4, "b": 5, "c": 6, "_stream": "s1"} 57 {"a": 7, "b": 8, "c": 9, "_stream": "s2"} 58 ``` 59 then `gather_val("log.jsonl", "s1", ("a", "b"))` will return 60 ```python 61 [ 62 [1, 2], 63 [4, 5] 64 ] 65 ``` 66 67 """ 68 data: list[dict] = jsonl_load_log(file) 69 70 output: list[list] = list() 71 72 for item in data: 73 # select for the stream 74 if ("_stream" in item) and (item["_stream"] == stream): 75 # select for the keys 76 if all(k in item for k in keys): 77 output.append(list(item[k] for k in keys)) 78 elif not allow_skip: 79 raise ValueError(f"missing keys '{keys = }' in '{item = }'") 80 81 return output
gather specific keys from a specific stream in a log file
example: if "log.jsonl" has contents:
{"a": 1, "b": 2, "c": 3, "_stream": "s1"}
{"a": 4, "b": 5, "c": 6, "_stream": "s1"}
{"a": 7, "b": 8, "c": 9, "_stream": "s2"}
then gather_val("log.jsonl", "s1", ("a", "b"))
will return
[
[1, 2],
[4, 5]
]