import os
import tempfile
import unittest
from unittest.mock import patch, MagicMock
from services.headlines import is_headline, prepare_headlines
from services.normalization import canonicalize, get_excluded_phrases, is_excluded
PHRASES = [
"skip to content",
"privacy choices",
"terms of service",
"privacy policy",
]
class TestCanonicalize(unittest.TestCase):
def test_lowercase_and_collapse(self):
self.assertEqual(canonicalize(" Terms-of-Service. "), "terms of service")
def test_empty(self):
self.assertEqual(canonicalize(""), "")
self.assertEqual(canonicalize(None), "")
class TestIsExcluded(unittest.TestCase):
def test_exact_match(self):
self.assertTrue(is_excluded("Terms of Service", PHRASES))
def test_sub_phrase_match(self):
# "privacy choices" matches the longer "Your Privacy Choices".
self.assertTrue(is_excluded("Your Privacy Choices", PHRASES))
def test_case_and_punctuation_insensitive(self):
self.assertTrue(is_excluded("SKIP-TO-CONTENT", PHRASES))
def test_no_match(self):
self.assertFalse(is_excluded("Federal Reserve holds rates steady", PHRASES))
def test_no_phrases(self):
self.assertFalse(is_excluded("anything here now", []))
self.assertFalse(is_excluded("anything here now", None))
class TestGetExcludedPhrases(unittest.TestCase):
def test_loads_skipping_comments_and_blanks(self):
path = None
try:
with tempfile.NamedTemporaryFile("w", suffix=".txt", delete=False) as f:
f.write("# a comment\n\nskip to content\n\nprivacy choices\n# another\n")
path = f.name
self.assertEqual(get_excluded_phrases(path), ["skip to content", "privacy choices"])
finally:
if path:
os.unlink(path)
class TestIsHeadlineExcluded(unittest.TestCase):
def setUp(self):
self.stopwords = {"the", "a", "an", "in", "on", "and", "of", "to"}
def test_boilerplate_rejected(self):
for text in ["Your Privacy Choices", "Skip to Content", "Terms of Service"]:
with self.subTest(text=text):
self.assertFalse(is_headline(text, self.stopwords, PHRASES))
def test_real_headline_kept(self):
self.assertTrue(is_headline("Senate passes major infrastructure bill", self.stopwords, PHRASES))
class TestPrepareHeadlinesExcluded(unittest.TestCase):
def setUp(self):
self.stopwords = {"the", "a", "an", "in", "on", "and", "of", "to"}
@patch("services.headlines.requests.get")
def test_boilerplate_links_are_filtered_out(self, mock_get):
html = (
""
"Skip to Content"
"Terms of Service"
"Your Privacy Choices"
"Senate passes major infrastructure bill"
""
)
mock_response = MagicMock()
mock_response.status_code = 200
mock_response.content = html.encode("utf-8")
mock_response.text = html
mock_get.return_value = mock_response
headlines = prepare_headlines(["https://example.com/news"], self.stopwords, excluded_phrases=PHRASES)
texts = [h.display_text for h in headlines]
self.assertEqual(texts, ["Senate passes major infrastructure bill"])
if __name__ == "__main__":
unittest.main()