Coverage for src/finbot/features/persistence.py: 80%

1import json 

2import re 

3from datetime import datetime 

4from pathlib import Path 

5 

6from finbot.features.core import ( 

7 FeatureDataset, 

8 FeatureDatasetMetadata, 

9 FeatureRow, 

10 FeatureSchema, 

11) 

12 

13FEATURE_DATASET_FORMAT_VERSION = 1 

14_DATASET_ID_PATTERN = re.compile(r"[0-9a-f]{64}") 

15 

16 

17class FeatureDatasetStore: 

18 def __init__(self, root: Path) -> None: 

19 self.root = root 

20 

21 def save(self, dataset: FeatureDataset) -> str: 

22 dataset_id = dataset.dataset_id 

23 payload = { 

24 "format_version": FEATURE_DATASET_FORMAT_VERSION, 

25 "dataset_id": dataset_id, 

26 "dataset": dataset.to_dict(), 

27 } 

28 serialized = json.dumps( 

29 payload, 

30 sort_keys=True, 

31 separators=(",", ":"), 

32 allow_nan=False, 

33 ) 

34 

35 self.root.mkdir(parents=True, exist_ok=True) 

36 destination = self.root / f"{dataset_id}.json" 

37 temporary = destination.with_suffix(".json.tmp") 

38 temporary.write_text(serialized, encoding="utf-8") 

39 temporary.replace(destination) 

40 return dataset_id 

41 

42 def load(self, dataset_id: str) -> FeatureDataset: 

43 if _DATASET_ID_PATTERN.fullmatch(dataset_id) is None: 

44 raise ValueError("Feature dataset ID must be a lowercase SHA-256 digest.") 

45 

46 path = self.root / f"{dataset_id}.json" 

47 try: 

48 payload = json.loads(path.read_text(encoding="utf-8")) 

49 except FileNotFoundError as exc: 

50 raise FileNotFoundError(f"Feature dataset {dataset_id} does not exist.") from exc 

51 except json.JSONDecodeError as exc: 

52 raise ValueError(f"Feature dataset {dataset_id} contains invalid JSON.") from exc 

53 

54 dataset = self._parse_payload(payload) 

55 if payload.get("dataset_id") != dataset_id or dataset.dataset_id != dataset_id: 

56 raise ValueError("Feature dataset identity does not match its contents.") 

57 return dataset 

58 

59 @staticmethod 

60 def _parse_payload(payload: object) -> FeatureDataset: 

61 if not isinstance(payload, dict): 

62 raise ValueError("Feature dataset payload must be an object.") 

63 if payload.get("format_version") != FEATURE_DATASET_FORMAT_VERSION: 

64 raise ValueError("Unsupported feature dataset format version.") 

65 

66 raw_dataset = payload.get("dataset") 

67 if not isinstance(raw_dataset, dict): 

68 raise ValueError("Feature dataset payload is missing its dataset object.") 

69 

70 feature_names = raw_dataset.get("feature_names") 

71 raw_metadata = raw_dataset.get("metadata") 

72 raw_rows = raw_dataset.get("rows") 

73 if not isinstance(feature_names, list) or not all( 73 ↛ 76line 73 didn't jump to line 76 because the condition on line 73 was never true

74 isinstance(name, str) and name for name in feature_names 

75 ): 

76 raise ValueError("Feature names must be a list of non-empty strings.") 

77 if not isinstance(raw_rows, list): 

78 raise ValueError("Feature dataset rows must be a list.") 

79 

80 rows = tuple( 

81 FeatureDatasetStore._parse_row(raw_row, len(feature_names)) 

82 for raw_row in raw_rows 

83 ) 

84 metadata = FeatureDatasetStore._parse_metadata(raw_metadata) 

85 if tuple(feature.name for feature in metadata.feature_schema) != tuple(feature_names): 

86 raise ValueError("Feature schema names must match feature names and ordering.") 

87 return FeatureDataset( 

88 feature_names=tuple(feature_names), 

89 rows=rows, 

90 metadata=metadata, 

91 ) 

92 

93 @staticmethod 

94 def _parse_metadata(raw_metadata: object) -> FeatureDatasetMetadata: 

95 if not isinstance(raw_metadata, dict): 95 ↛ 96line 95 didn't jump to line 96 because the condition on line 95 was never true

96 raise ValueError("Feature dataset metadata must be an object.") 

97 

98 source_dataset_id = raw_metadata.get("source_dataset_id") 

99 generator_version = raw_metadata.get("generator_version") 

100 raw_schema = raw_metadata.get("feature_schema") 

101 if not isinstance(source_dataset_id, str) or not source_dataset_id: 101 ↛ 102line 101 didn't jump to line 102 because the condition on line 101 was never true

102 raise ValueError("Source dataset ID must be a non-empty string.") 

103 if ( 103 ↛ 108line 103 didn't jump to line 108 because the condition on line 103 was never true

104 isinstance(generator_version, bool) 

105 or not isinstance(generator_version, int) 

106 or generator_version < 1 

107 ): 

108 raise ValueError("Feature generator version must be a positive integer.") 

109 if not isinstance(raw_schema, list): 109 ↛ 110line 109 didn't jump to line 110 because the condition on line 109 was never true

110 raise ValueError("Feature schema must be a list.") 

111 

112 return FeatureDatasetMetadata( 

113 source_dataset_id=source_dataset_id, 

114 feature_schema=tuple( 

115 FeatureDatasetStore._parse_schema_entry(entry) for entry in raw_schema 

116 ), 

117 generator_version=generator_version, 

118 ) 

119 

120 @staticmethod 

121 def _parse_schema_entry(raw_entry: object) -> FeatureSchema: 

122 if not isinstance(raw_entry, dict): 122 ↛ 123line 122 didn't jump to line 123 because the condition on line 122 was never true

123 raise ValueError("Feature schema entry must be an object.") 

124 name = raw_entry.get("name") 

125 parameters = raw_entry.get("parameters") 

126 if not isinstance(name, str) or not name: 126 ↛ 127line 126 didn't jump to line 127 because the condition on line 126 was never true

127 raise ValueError("Feature schema name must be a non-empty string.") 

128 if not isinstance(parameters, dict) or not all( 128 ↛ 134line 128 didn't jump to line 134 because the condition on line 128 was never true

129 isinstance(key, str) 

130 and key 

131 and (value is None or isinstance(value, (str, int, float, bool))) 

132 for key, value in parameters.items() 

133 ): 

134 raise ValueError("Feature schema parameters must contain scalar values.") 

135 return FeatureSchema( 

136 name=name, 

137 parameters=tuple( 

138 (key, value) for key, value in parameters.items() 

139 ), # type: ignore[misc] 

140 ) 

141 

142 @staticmethod 

143 def _parse_row(raw_row: object, feature_count: int) -> FeatureRow: 

144 if not isinstance(raw_row, dict): 

145 raise ValueError("Feature dataset row must be an object.") 

146 

147 symbol = raw_row.get("symbol") 

148 timestamp = raw_row.get("timestamp") 

149 values = raw_row.get("values") 

150 if not isinstance(symbol, str) or not symbol: 150 ↛ 151line 150 didn't jump to line 151 because the condition on line 150 was never true

151 raise ValueError("Feature dataset row symbol must be a non-empty string.") 

152 if not isinstance(timestamp, str): 152 ↛ 153line 152 didn't jump to line 153 because the condition on line 152 was never true

153 raise ValueError("Feature dataset row timestamp must be an ISO datetime string.") 

154 if not isinstance(values, list) or len(values) != feature_count: 154 ↛ 155line 154 didn't jump to line 155 because the condition on line 154 was never true

155 raise ValueError("Feature dataset row values must match the feature count.") 

156 if any( 156 ↛ 161line 156 didn't jump to line 161 because the condition on line 156 was never true

157 value is not None 

158 and (isinstance(value, bool) or not isinstance(value, (int, float))) 

159 for value in values 

160 ): 

161 raise ValueError("Feature dataset values must be numbers or null.") 

162 

163 try: 

164 parsed_timestamp = datetime.fromisoformat(timestamp) 

165 except ValueError as exc: 

166 raise ValueError("Feature dataset row contains an invalid timestamp.") from exc 

167 

168 return FeatureRow( 

169 symbol=symbol, 

170 timestamp=parsed_timestamp, 

171 values=tuple(None if value is None else float(value) for value in values), 

172 )