import logging
import re
import requests
from services.dates import parse_datetime
from services.normalization import normalize_headline
from structs.headline import Headline
logger = logging.getLogger(__name__)
DEFAULT_TIMEOUT = 10
MIN_HEADLINE_WORDS = 3
# Single streaming pass over the HTML: captures " # group 1 attrs, group 2 text
r"|(?:(?:datetime|dateTime)\s*=\s*[\"']([^\"']+)[\"'])" # group 3 datetime attr
r"|<(?:article|li)\b[^>]*>" # container boundary (no group)
r"|<(?:a|span)\b[^>]*>\s*([^<]*?)\s*", # group 4 headline text
re.IGNORECASE | re.DOTALL,
)
def _extract_candidates(source_content):
"""Yield ``(text, published_at)`` tuples in document order.
Each headline is associated with the most recent publication timestamp seen
before it. Timestamps reset at each ````/``
`` boundary so a
headline with no date of its own does not inherit another story's timestamp.
"""
candidates = []
last_published = None
for match in _CANDIDATE_RE.finditer(source_content):
if match.group(1) is not None:
# A element: prefer an explicit datetime/title
# attribute, otherwise fall back to its inner text.
attrs = match.group(1)
attr_match = re.search(r'(?:datetime|dateTime)\s*=\s*[\"\']([^\"\']+)[\"\']', attrs, re.IGNORECASE) \
or re.search(r'title\s*=\s*[\"\']([^\"\']+)[\"\']', attrs, re.IGNORECASE)
raw = attr_match.group(1) if attr_match else match.group(2)
parsed = parse_datetime(raw)
if parsed is not None:
last_published = parsed
elif match.group(3) is not None:
# A datetime attribute on some non-