data = {
    'cb': 'visitor_gray_callback',
    'tid': '',
    'new_tid': 'null'
}

resp = session.post(url, headers=headers, data=data)
match = re.search(r'visitor_gray_callback\((.*)\)', resp.text)
json_data = json.loads(match.group(1))
sub = json_data['data']['sub']
session.cookies.set('SUB', sub, domain='.weibo.cn')

Header['x-xsrf-token'] == Cookie['XSRF-TOKEN']

# -*- coding: utf-8 -*-
import requests
import json
import re

class WeiboH5VisitorAuth:
    def __init__(self):
        self.session = requests.Session()
        self._init_headers()
        self.sub = None
        self.xsrf_token = None

    def _init_headers(self):
        """Configure browser fingerprint"""
        headers_base = {
            'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36',
            'sec-ch-ua': '"Chromium";v="142", "Microsoft Edge";v="142"',
            'sec-ch-ua-mobile': '?0',
            'sec-ch-ua-platform': '"Windows"',
            'Accept-Encoding': 'gzip, deflate, br, zstd',
            'dnt': '1',
        }
        self.session.headers.update(headers_base)

    def step1_obtain_identity_token(self):
        """Phase 1: Obtain SUB"""
        url = 'https://visitor.passport.weibo.cn/visitor/genvisitor2'
        headers = {
            'Content-Type': 'application/x-www-form-urlencoded',
            'Origin': 'https://visitor.passport.weibo.cn',
        }
        data = {'cb': 'visitor_gray_callback', 'tid': '', 'new_tid': 'null'}

        resp = self.session.post(url, headers=headers, data=data)
        match = re.search(r'visitor_gray_callback\((.*)\)', resp.text)
        json_data = json.loads(match.group(1))

        if json_data.get('retcode') == 20000000:
            self.sub = json_data['data']['sub']
            self.session.cookies.set('SUB', self.sub, domain='.weibo.cn')
            self.session.cookies.set('SUBP', json_data['data']['subp'], domain='.weibo.cn')

    def step2_initialize_session(self):
        """Phase 2: Obtain XSRF-TOKEN"""
        url = 'https://m.weibo.cn'
        headers = {'Referer': 'https://visitor.passport.weibo.cn/'}

        self.session.get(url, headers=headers)
        self.xsrf_token = self.session.cookies.get('XSRF-TOKEN')

    def step3_access_api(self, container_id):
        """Phase 3: Access API"""
        api_url = 'https://m.weibo.cn/api/container/getIndex'
        params = {'containerid': container_id, 'page': 1, 'count': 10}
        headers = {
            'Accept': 'application/json, text/plain, */*',
            'x-xsrf-token': self.xsrf_token,  # Double Submit
            'Referer': f'https://m.weibo.cn/u/{container_id}'
        }

        resp = self.session.get(api_url, params=params, headers=headers)
        return resp.json()

    def run(self, target_id):
        self.step1_obtain_identity_token()
        self.step2_initialize_session()
        return self.step3_access_api(target_id)

class Auth:
    def __init__(self, config):
        self.use_cookie = config.get("cookies", {}).get("enabled", False)
        self._visitor_cookies = {}

    async def init_visitor_auth(self, session):
        """Cookie-free scheme: Initialize visitor authentication"""
        # Implement three-phase authentication
        pass

class Spider:
    async def fetch(self, page=0):
        # Try the Cookie scheme first
        if self.auth.use_cookie and self.auth._get_cookie():
            try:
                return await self._fetch_with_cookie(page)
            except Exception as e:
                logger.warning(f"Cookie scheme failed: {e}, switching to Cookie-free scheme")

        # Fallback to Cookie-free scheme
        return await self._fetch_without_cookie(page)

{
  "ok": 1,
  "data": {
    "list": [{
      "mblogid": "xxx",
      "text_raw": "Plain text",
      "pic_infos": {...}
    }]
  }
}

{
  "ok": 1,
  "data": {
    "cards": [{
      "card_group": [{
        "mblog": {
          "mid": "xxx",
          "text": "<br />HTML text<br />",
          "pics": [...]
        }
      }]
    }]
  }
}

def _parse_mobile_mblogs(self, data):
    for card in data.get("cards", []):
        for item in card.get("card_group", []):
            mblog = item["mblog"]
            # 1. Convert <br> to newlines
            text_raw = re.sub(r'<br\s*/?>', '\n', mblog.get("text", ""))
            # 2. Strip other HTML tags
            text_raw = re.sub(r'<[^>]+>', '', text_raw).strip()

            self._results.append(Blog(
                mblogid=mblog.get("mid"),
                text_raw=text_raw,
                is_long_text=mblog.get("isLongText", False),
                use_mobile_api=True
            ))

async def fetch_long_text(self, client, auth):
    if not self.is_long_text:
        return self.text_raw

    if self.use_mobile_api:
        api = "https://m.weibo.cn/statuses/extend"
    else:
        api = "https://weibo.com/ajax/statuses/longtext"

    response = await client.get(api, params={"id": self.mblogid})
    long_text = response.json()["data"]["longTextContent"]

    # Clean HTML and preserve newlines
    long_text = re.sub(r'<br\s*/?>', '\n', long_text)
    return re.sub(r'<[^>]+>', '', long_text).strip()

class Spider:
    def __init__(self):
        self._client = None  # Save client reference

    async def _fetch_without_cookie(self, page):
        self._client = httpx.AsyncClient()
        await self.auth.init_visitor_auth(self._client)
        # Fetch list...
        return self  # Keep client open

# In SkyDaily.get_daily_data
try:
    await spider.fetch()
    blog = spider.filter_by_regex(pattern).one()

    # Reuse spider's client to fetch long text
    if spider._client:
        text = await blog.fetch_long_text(spider._client, auth=auth)
finally:
    # Ensure client is closed
    if spider._client:
        await spider._client.aclose()

<span class="surl-text">sky光遇超话</span> 11.22 | 每日任务

sky光遇超话 11.22 | 每日任务

# Old regex (relies on # tags)
pattern = r"^#[^#]*光遇[^#]*超话]#\s*\d{1,2}\.\d{1,2}\s*"

# New regex (keyword matching)
pattern = r".*sky光遇.*每日任务.*"

{
  "cookies": {
    "enabled": false,
    "sub": "",
    "xsrf_token": ""
  },
  "data_sources": [
    "7360748659:.*sky光遇.*每日任务.*:今天游离翻车了吗",
    "5539106873:^【国服·每日任务攻略】:陈陈努力不鸽"
  ]
}

async def fetch(self, page=0, max_attempts=3, retry_delay=2):
    for attempt in range(max_attempts):
        try:
            # Attempt to fetch data
            pass
        except (httpx.HTTPStatusError, httpx.TimeoutException) as e:
            if attempt < max_attempts - 1:
                logger.warning(f"Request failed, retrying in {retry_delay} seconds...")
                await asyncio.sleep(retry_delay)
                continue
            else:
                raise
        finally:
            # Ensure resources are cleaned up
            if self._client:
                await self._client.aclose()

[INFO] Using Cookie-free scheme to fetch Weibo data
[INFO] Visitor authentication initialized successfully
[INFO] Retrieved 1 post from today's source
[INFO] Today's server strategy query complete, successfully fetched 2/2 data sources