Spaces:
Running
Running
File size: 1,025 Bytes
1fbc47b d18b34d 1fbc47b |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 |
import unittest
from src.data.preprocessing import TextPreprocessor
from src.data.tokenization import Tokenizer, TokenizerConfig
class _StubTokenizer(Tokenizer):
def __init__(self, max_length: int) -> None:
# Avoid expensive huggingface initialisation by skipping super().__init__
self.config = TokenizerConfig(max_length=max_length)
def batch_encode(self, texts, *, max_length=None):
raise NotImplementedError
class TextPreprocessorTests(unittest.TestCase):
def test_matching_max_length_leaves_tokenizer_unchanged(self) -> None:
tokenizer = _StubTokenizer(max_length=128)
TextPreprocessor(tokenizer=tokenizer, max_length=128)
self.assertEqual(tokenizer.config.max_length, 128)
def test_conflicting_max_length_raises_value_error(self) -> None:
tokenizer = _StubTokenizer(max_length=256)
with self.assertRaises(ValueError):
TextPreprocessor(tokenizer=tokenizer, max_length=128)
if __name__ == "__main__":
unittest.main()
|