import csv, requests
from bs4 import BeautifulSoup

rows = []
for url in source_urls:
    soup = BeautifulSoup(requests.get(url, timeout=10).text, "html.parser")
    rows.append({
        "url": url,
        "title": soup.select_one("h1").get_text(strip=True),
        "text": soup.select_one(".article").get_text("\n", strip=True),
        "source": "某来源",
    })

with open("articles.csv", "w", newline="", encoding="utf-8") as f:
    csv.DictWriter(f, fieldnames=["url", "title", "text", "source"]).writerows(rows)

import os, json, hanlp
os.environ["HANLP_HOME"] = "./hanlp_models"   # 模型首次运行会自动下载

HanLP = hanlp.load(
    hanlp.pretrained.mtl.CLOSE_TOK_POS_NER_SRL_DEP_SDP_CON_ELECTRA_SMALL_ZH,
    tasks=['tok', 'pos', 'ner'])

texts = ["省委领导赴齐齐哈尔市泰来县调研防汛抗旱与备春耕工作。",
         "黑龙江省将大力发展数字龙江，深化对俄经贸合作。"]

with open("heilongjiang_nlp_data.jsonl", "w", encoding="utf-8") as f:
    for i, text in enumerate(texts):
        doc = HanLP(text)
        rec = {
            "id": i + 1,
            "original_text": text,
            "tokens": doc['tok/fine'],      # 分词列表
            "pos_tags": doc['pos/ctb'],     # 词性列表（NN/NR/NT/NS…）
            "ner_entities": doc['ner/msra'] # 命名实体：地名/机构名/人名…
        }
        f.write(json.dumps(rec, ensure_ascii=False) + "\n")

import json, numpy as np
from scipy import sparse
from sklearn.feature_extraction.text import TfidfVectorizer

corpus, original_texts = [], []
with open("heilongjiang_nlp_data.jsonl", encoding="utf-8") as f:
    for line in f:
        rec = json.loads(line)
        original_texts.append(rec["original_text"])
        # 核心：只保留名词（词性以 N 开头），动词/形容词/标点全丢
        valid = [w for w, p in zip(rec["tokens"], rec["pos_tags"]) if p.startswith('N')]
        corpus.append(" ".join(valid))

vectorizer = TfidfVectorizer(max_df=0.85, min_df=2, max_features=10000)
tfidf_matrix = vectorizer.fit_transform(corpus)
feature_names = np.array(vectorizer.get_feature_names_out())

sparse.save_npz("tfidf_matrix.npz", tfidf_matrix)  # 给阶段 4 的 LDA 用

# 每篇取 TF-IDF 分数最高的前 5 个词作为标签
with open("heilongjiang_tagged_database.jsonl", "w", encoding="utf-8") as f:
    for i in range(tfidf_matrix.shape[0]):
        scores = tfidf_matrix[i].toarray()[0]
        top = [feature_names[j] for j in scores.argsort()[-5:][::-1] if scores[j] > 0]
        f.write(json.dumps({"id": i+1, "text": original_texts[i], "tags": top},
                           ensure_ascii=False) + "\n")

import json, numpy as np, pandas as pd
from scipy import sparse
from sklearn.decomposition import LatentDirichletAllocation

tfidf_matrix = sparse.load_npz("tfidf_matrix.npz")
feature_names = np.array(json.load(open("tfidf_feature_names.json", encoding="utf-8")))

# Alpha/Beta 调小 → 每篇文章主题更单一、每个主题词更集中
lda = LatentDirichletAllocation(
    n_components=50, doc_topic_prior=0.01, topic_word_prior=0.01,
    max_iter=30, learning_method='online', random_state=42, n_jobs=-1)
doc_topic_dist = lda.fit_transform(tfidf_matrix)

# 主题清单：每个主题取权重最高的 15 个词
topic_results = []
for idx, comp in enumerate(lda.components_):
    top_words = [feature_names[i] for i in comp.argsort()[:-16:-1]]
    topic_results.append({"主题编号": idx, "核心关键词": " / ".join(top_words)})
pd.DataFrame(topic_results).to_csv("lda_50_topics_summary.csv", index=False, encoding="utf-8-sig")

# 文章 → 主题 映射：每篇归到概率最高的主题
with open("final_articles_with_topics.jsonl", "w", encoding="utf-8") as f:
    for i, art in enumerate(articles_data):
        art["topic_id"] = int(doc_topic_dist[i].argmax())
        f.write(json.dumps(art, ensure_ascii=False) + "\n")

from itertools import zip_longest
import json

with open(BASE) as fb, open(TOPIC) as ft, open(OUT, "w", encoding="utf-8") as fo:
    for idx, (bl, tl) in enumerate(zip_longest(fb, ft, fillvalue=None), 1):
        if bl is None or tl is None:
            raise SystemExit(f"行数不匹配，第 {idx} 行一方提前结束")  # 不静默错位
        base = json.loads(bl)
        base["topic_id"] = json.loads(tl).get("topic_id")  # 只追加主题字段
        fo.write(json.dumps(base, ensure_ascii=False) + "\n")

conn = sqlite3.connect("merged_articles.db")
conn.execute("DROP TABLE IF EXISTS articles")
conn.execute("""CREATE TABLE articles (
    url TEXT PRIMARY KEY, id INTEGER, title TEXT, text TEXT,
    source TEXT, tags TEXT, topic_name TEXT)""")
conn.execute("CREATE INDEX IF NOT EXISTS idx_topic_name ON articles(topic_name)")

seen_urls, seen_md5, title_prefixes = set(), set(), {}

for rec in iter_jsonl(BASE):
    url = rec.get("url", "")
    if url in seen_urls: continue            # ① url 重复
    seen_urls.add(url)
    text = rec.get("text", "")
    md5 = hashlib.md5(text.encode()).hexdigest()
    if md5 in seen_md5: continue             # ② 正文完全相同
    title, prefix = rec.get("title", ""), text[:200]
    if title and prefix:
        for p in title_prefixes.get(title, []):
            if difflib.SequenceMatcher(None, prefix, p).ratio() >= 0.85:  # ③ 前200字相似
                break
        else:
            title_prefixes.setdefault(title, []).append(prefix)
        else:
            continue
    seen_md5.add(md5)
    # topic_id → 主题名称（经 summary.csv 映射），直接存名字不存数字编号
    topic_name = topic_to_group.get(url_to_topic.get(url, -1), "")
    conn.execute(
        "INSERT OR REPLACE INTO articles (url,id,title,text,source,tags,topic_name) VALUES (?,?,?,?,?,?,?)",
        (url, rec.get("id"), rec.get("title"), text, rec.get("source"),
         json.dumps(rec.get("tags", []), ensure_ascii=False), topic_name))
conn.commit()

cur.execute("DELETE FROM articles WHERE topic_name = '删掉'")
cur.execute("UPDATE articles SET topic_name='民生福祉' WHERE topic_name IN ('民生服务','民生福祉')")
cur.execute("UPDATE articles SET topic_name='生态环境' WHERE topic_name IN ('林业生态','生态环境')")
cur.execute("UPDATE articles SET topic_name='中央精神' WHERE topic_name IN ('省委精神','中央精神')")
cur.execute("UPDATE articles SET topic_name='经济指数' WHERE topic_name IN ('经济科技','经济指数')")
conn.commit()