Coverage for muutils/logger/log_util.py: 0%

1from muutils.jsonlines import jsonl_load_log 

2 

3 

4def get_any_from_stream(stream: list[dict], key: str) -> None: 

5 """get the first value of a key from a stream. errors if not found""" 

6 for msg in stream: 

7 if key in msg: 

8 return msg[key] 

9 

10 raise KeyError(f"key '{key}' not found in stream") 

11 

12 

13def gather_log(file: str) -> dict[str, list[dict]]: 

14 """gathers and sorts all streams from a log""" 

15 data: list[dict] = jsonl_load_log(file) 

16 output: dict[str, list[dict]] = dict() 

17 

18 for item in data: 

19 stream: str = item.get("_stream", None) 

20 if stream not in output: 

21 output[stream] = list() 

22 output[stream].append(item) 

23 

24 return output 

25 

26 

27def gather_stream( 

28 file: str, 

29 stream: str, 

30) -> list[dict]: 

31 """gets all entries from a specific stream in a log file""" 

32 data: list[dict] = jsonl_load_log(file) 

33 

34 output: list[dict] = list() 

35 

36 for item in data: 

37 # select for the stream 

38 if ("_stream" in item) and (item["_stream"] == stream): 

39 output.append(item) 

40 return output 

41 

42 

43def gather_val( 

44 file: str, 

45 stream: str, 

46 keys: tuple[str], 

47 allow_skip: bool = True, 

48) -> list[list]: 

49 """gather specific keys from a specific stream in a log file 

50 

51 example: 

52 if "log.jsonl" has contents: 

53 ```jsonl 

54 {"a": 1, "b": 2, "c": 3, "_stream": "s1"} 

55 {"a": 4, "b": 5, "c": 6, "_stream": "s1"} 

56 {"a": 7, "b": 8, "c": 9, "_stream": "s2"} 

57 ``` 

58 then `gather_val("log.jsonl", "s1", ("a", "b"))` will return 

59 ```python 

60 [ 

61 [1, 2], 

62 [4, 5] 

63 ] 

64 ``` 

65 

66 """ 

67 data: list[dict] = jsonl_load_log(file) 

68 

69 output: list[list] = list() 

70 

71 for item in data: 

72 # select for the stream 

73 if ("_stream" in item) and (item["_stream"] == stream): 

74 # select for the keys 

75 if all(k in item for k in keys): 

76 output.append(list(item[k] for k in keys)) 

77 elif not allow_skip: 

78 raise ValueError(f"missing keys '{keys = }' in '{item = }'") 

79 

80 return output