Update app.py

app.py

@@ -1,4 +1,3 @@
-# app.py
 import os
 import spacy
 import stanza
@@ -13,8 +12,6 @@ import streamlit as st
 import io
 from newspaper import Article
 import concurrent.futures
-import json
-import tempfile
 
 # ===============================
 # 🔑 Vertex AI Setup
@@ -22,6 +19,9 @@ import tempfile
 import vertexai
 from vertexai.preview.generative_models import GenerativeModel
 
+import json
+import tempfile
+
 # Ensure GCP credentials exist
 if "GCP_SERVICE_ACCOUNT_JSON" not in os.environ:
     raise RuntimeError("❌ GCP_SERVICE_ACCOUNT_JSON secret not found in Hugging Face Space")
@@ -119,19 +119,14 @@ def load_pipelines(language_code):
     return emotion_pipeline, sentiment_pipeline
 
 # ===============================
-# DOCX Reader (enhanced: detect bold runs as sub-headlines)
+# DOCX Reader
 # ===============================
 def read_and_split_articles(file_path):
     doc = docx.Document(file_path)
     paragraphs = []
     for para in doc.paragraphs:
-        text = para.text.strip()
-        if not text:
-            continue
-
-        # If this paragraph contains bold run(s), keep it as its own paragraph (likely a subhead)
-        is_bold = any([r.bold for r in para.runs]) if para.runs else False
-        paragraphs.append(text if not is_bold else text)
+        if para.text.strip():
+            paragraphs.append(para.text.strip())
 
     headline = paragraphs[0] if paragraphs else ""
     body_paragraphs = paragraphs[1:] if len(paragraphs) > 1 else []
@@ -139,24 +134,18 @@ def read_and_split_articles(file_path):
     return headline, body_paragraphs
 
 # ===============================
-# Robust URL Reader
+# URL Reader
 # ===============================
 def read_article_from_url(url):
-    # safe-guard trailing spaces and encoded spaces
-    if not url or not isinstance(url, str):
-        return "", []
-    url = url.strip()
-    # try to download & parse
-    try:
-        article = Article(url)
-        article.download()
-        article.parse()
-    except Exception as e:
-        st.warning(f"⚠️ Could not download article: {e}")
-        return "", []
+    article = Article(url)
+    article.download()
+    article.parse()
+
     headline = article.title.strip() if article.title else ""
     text_body = article.text.strip() if article.text else ""
+
     body_paragraphs = [p.strip() for p in text_body.split("\n") if p.strip()]
+
     return headline, body_paragraphs
 
 # ===============================
@@ -164,19 +153,20 @@ def read_article_from_url(url):
 # ===============================
 def filter_neutral(emotion_results, neutral_threshold=0.75):
     sorted_results = sorted(emotion_results, key=lambda x: x["score"], reverse=True)
+
     scores = {}
     for r in sorted_results:
         scores[r["label"]] = round(r["score"], 3)
+
     if "neutral" in scores and scores["neutral"] > neutral_threshold:
         scores.pop("neutral")
+
     return scores
 
 # ===============================
 # Split Sentences
 # ===============================
 def split_sentences(text, lang):
-    if not text:
-        return []
     if lang == "hi":
         sentences = re.split(r"।", text)
         return [s.strip() for s in sentences if s.strip()]
@@ -216,203 +206,82 @@ def get_pos_tags(sentence, lang):
 def normalize_scores(scores: dict):
     if not scores:
         return scores
+
     max_val = max(scores.values())
     if max_val == 0:
         return scores
+
     normalized = {}
     for k, v in scores.items():
         normalized[k] = round(v / max_val, 3)
+
     return normalized
 
 # ===============================
-# Enhanced Keyword extraction for SEO suggestions
-# ===============================
-COMMON_STOPWORDS = set([
-    "the","a","an","and","or","in","on","to","of","for","is","it","that","this","with","its","as",
-    "are","be","was","were","by","from","at","have","has","had","but","not","which","who","what",
-    "when","where","why","how","will","can","should","our","we","you","your","I","they","their","his","her"
-])
-
-def extract_keywords(text, top_n=6):
-    # simple heuristic: words length > 3, not stopwords, prefer words appearing earlier & freq > 1
-    if not text:
-        return []
-    tokens = re.findall(r"\w+", text.lower())
-    tokens = [t for t in tokens if t not in COMMON_STOPWORDS and len(t) > 3]
-    freq = Counter(tokens)
-    if not freq:
-        return []
-    # prefer words that appear in first 80 chars (title/head) or start of article
-    head_tokens = set(re.findall(r"\w+", text[:200].lower()))
-    scored = []
-    for word, count in freq.items():
-        score = count
-        if word in head_tokens:
-            score += 1.25
-        scored.append((word, score))
-    scored.sort(key=lambda x: x[1], reverse=True)
-    keywords = [w for w,_ in scored[:top_n]]
-    return keywords
-
-def shorten_headline_variants(headline):
-    # provide 1-3 short headline ideas (heuristic)
-    words = headline.split()
-    variants = []
-    # variant 1: take first 6-8 words
-    variants.append(" ".join(words[:8]) + ("..." if len(words) > 8 else ""))
-    # variant 2: main noun + core keyword (if any)
-    kws = extract_keywords(headline, top_n=3)
-    if kws:
-        variants.append(f"{kws[0].capitalize()}: {words[0]} {' '.join(words[1:4])}")
-    # unique variant fallback: remove stopwords from headline
-    variants.append(" ".join([w for w in words if w.lower() not in COMMON_STOPWORDS])[:70])
-    # dedupe & cleanup
-    clean = []
-    for v in variants:
-        v = v.strip()
-        if v and v not in clean:
-            clean.append(v)
-    return clean[:3]
-
-# ===============================
-# Compute SEO suggestions (single place; lightweight heuristics)
-# ===============================
-def compute_seo_suggestions_enhanced(headline, paragraphs, top_n_keywords=5):
-    """
-    Returns small dict:
-    {
-      "keywords": [...],
-      "keyword_density": {...},
-      "suggestions": [...],
-      "headline_suggestions": [...]
-    }
-    Show main SEO block only once (headline area).
-    """
-    text = (headline or "") + " " + " ".join(paragraphs or [])
-    tokens = re.findall(r"\w+", text.lower())
-    tokens = [t for t in tokens if t not in COMMON_STOPWORDS and len(t) > 3]
-    freq = Counter(tokens)
-    total = sum(freq.values()) or 1
-    # pick top keywords but filter random garbage by requiring either freq>1 or appearing in headline
-    top = []
-    for w, c in freq.most_common(30):
-        if c > 1 or (headline and w in headline.lower()):
-            top.append((w, c))
-        if len(top) >= top_n_keywords:
-            break
-    keywords = [k for k,_ in top]
-    keyword_density = {k: round(freq[k] / total, 4) for k in keywords}
-    suggestions = []
-    # headline length advice
-    if headline:
-        if len(headline) > 70:
-            suggestions.append("Headline is long (>70 chars). Consider shortening to 50–65 chars for better CTR.")
-        elif len(headline) < 30:
-            suggestions.append("Headline is short (<30 chars). Consider adding a descriptive keyword for clarity/SEO.")
-    # keyword placement
-    if keywords:
-        suggestions.append(f"Primary keywords to consider: {', '.join(keywords)}.")
-        # ensure at least one primary keyword in first 100 chars
-        head_sample = text[:100].lower()
-        if not any(k in head_sample for k in keywords[:2]):
-            suggestions.append("Consider including 1–2 primary keywords in the headline or first 100 words.")
-        # density advice (only sensible extremes)
-        for k, d in keyword_density.items():
-            if d < 0.003:
-                suggestions.append(f"Keyword '{k}' has low density ({d}). Consider using it once in the first 100 words.")
-            elif d > 0.06:
-                suggestions.append(f"Keyword '{k}' has high density ({d}). Review for possible keyword stuffing.")
-    # meta draft
-    body_tokens = [t for t in tokens]
-    meta = " ".join(body_tokens[:25])[:155].strip()
-    suggestions.append(f"Suggested meta (draft): {meta}...")
-    # headline ideas
-    headline_suggestions = shorten_headline_variants(headline) if headline else []
-    return {"keywords": keywords, "keyword_density": keyword_density, "suggestions": suggestions, "headline_suggestions": headline_suggestions}
-
-# ===============================
-# Improved Paragraph Cleaner
+# Clean Paragraphs (remove embeds/promos)
 # ===============================
 def clean_paragraphs(paragraphs):
-    """
-    - Merge bullets and numbered lists with previous paragraphs.
-    - Remove promotional or repetitive boilerplate.
-    - Detect and merge short fragments into previous paragraph.
-    """
     cleaned = []
-    prev = None
 
-    for raw_para in paragraphs:
-        if raw_para is None:
-            continue
-        text = raw_para.strip()
+    for para in paragraphs:
+        text = para.strip()
         if not text:
             continue
 
         upper_text = text.upper()
 
-        # skip known promo patterns
         if upper_text.startswith(("ALSO READ", "READ ALSO", "TRENDING", "MUST READ")):
             continue
-        if "और पढ़ें" in text or "यह भी पढ़ें" in text or "पूरा पढ़ें" in text:
-            continue
-
-        # skip obvious single-word labels like "PHOTO" or "VIDEO"
-        if len(text.split()) <= 2 and text.isupper():
-            continue
-
-        # if line looks like a bullet or numbered list or very short fragment,
-        # merge with previous paragraph instead of treating as its own paragraph
-        is_bullet = bool(re.match(r"^(\-|\•|\*|\d+[\.\)]\s)", text))
-        short_fragment = len(text.split()) < 6 and not text.endswith((".", "?", "!", ":"))
 
-        if (is_bullet or short_fragment) and prev is not None:
-            # merge into previous paragraph with a space
-            prev = prev.rstrip() + " " + text
-            cleaned[-1] = prev
+        if "और पढ़ें" in text or "यह भी पढ़ें" in text or "पूरा पढ़ें" in text:
             continue
 
-        # skip tiny key:value lines (promos)
-        if len(text.split()) < 5 and ":" in text and not text.endswith("?"):
+        if len(text.split()) < 5 and ":" in text:
             continue
 
-        # otherwise treat as a normal paragraph
         cleaned.append(text)
-        prev = text
 
     return cleaned
 
 # ===============================
-# Gemini Insight Generation (humanized prompts + gemini scoring + guardrails)
+# Gemini Insight Generation (patched with guardrails + snippet rewrites)
+# ===============================
+# ===============================
+# Gemini Insight Generation (patched with guardrails + snippet rewrites + Gemini emotions/sentiment)
+# ===============================
+# ===============================
+# Gemini Insight Generation (no Top 3 emotions, skip Gemini scoring if no rewrite)
+# ===============================
+# ===============================
+# Gemini Insight Generation (only Gemini sentiment + top 3 emotions)
+# ===============================
+# ===============================
+# Gemini Insight Generation (only Gemini sentiment + top 3 emotions, with context scoring)
 # ===============================
 def generate_insight(text, emotions, sentiment, level="Paragraph", emotion_pipeline=None, sentiment_pipeline=None):
-    """
-    - Calls Gemini to propose a *snippet* rewrite (word/phrase/sentence).
-    - Humanizes Gemini prompt and asks for Original → Rewrite → Why.
-    - Re-scores Gemini's rewrite using local pipelines (emotion + sentiment).
-    - Applies guardrails based on Gemini output (NOT original).
-    - Returns gemini_emotions (top-3 dict) and final_text string for display.
-    """
     try:
-        # Human-like Gemini prompt: ask for a specific snippet and human-sounding rewrite
+        # Always ask Gemini
         prompt = f"""
-You are a seasoned human editor. Use a natural, conversational tone — not robotic.
-
+You are a seasoned human editor with a natural, conversational tone — not robotic or formulaic.
 Text to review:
 {text}
-
 Task:
-- Identify the *specific phrase or sentence* that can be improved (only the smallest necessary span).
-- Output exactly in this format:
-Original → <the exact part to change>
-Rewrite → <a short, natural, human-sounding rewrite>
-Why → <one short sentence explaining the edit>
-
-If no rewrite is needed, output exactly:
+- Identify the *specific phrase or sentence* that can be improved for clarity, tone, or impact.
+- Present it as:
+Original → [the exact part]
+Rewrite → [a natural, human-sounding rewrite — avoid over-polishing or AI tone]
+Why → [briefly explain the edit as if giving human feedback — e.g., “This reads more fluidly” or “Helps it sound more direct.”]
+Guidelines:
+- Use everyday phrasing and mild imperfections that feel authentic.
+- Avoid mechanical transitions like “Overall,” “In summary,” or “This small change.”
+- Vary sentence rhythm and tone to mimic human writing.
+- Keep rewrites short and organic, not overly polished.
+- If the text is already fine, say exactly:
 No rewrite needed. The {level.lower()} reads naturally and clearly.
 """
+
+
         response_text = None
-        # Try Pro first, then Flash fallback
         for model_id, timeout in [
             ("publishers/google/models/gemini-2.5-pro", 40),
             ("publishers/google/models/gemini-2.5-flash", 25),
@@ -435,60 +304,46 @@ No rewrite needed. The {level.lower()} reads naturally and clearly.
         if not response_text:
             return {}, f"⚠️ No insight generated."
 
-        # If Gemini declines rewrite
+        # If Gemini says no rewrite → just show that (no extra scoring)
         if response_text.startswith("No rewrite needed"):
-            # return clear "no rewrite" phrasing so editorial doesn't get scary warnings
-            return {}, f"✅ No rewrite needed. The {level.lower()} reads naturally and clearly."
+            return {}, f"✅ {response_text}"
 
-        # Re-score Gemini output using a context (Original + Rewrite) so that emotion+sentiment reflect the suggested change
-        gemini_emotions = {}
-        gemini_sentiment = {}
+        # Otherwise, re-score Gemini rewrite using context (Original + Rewrite)
+        gemini_emotions, gemini_sentiment = {}, {}
         if emotion_pipeline is not None and sentiment_pipeline is not None:
             context_for_scoring = f"Original: {text}\nRewrite: {response_text}"
+
             emo_res_new = emotion_pipeline(context_for_scoring[:512])[0]
             gemini_emotions = filter_neutral(emo_res_new)
-            # keep top 3 emotions with scores
             sorted_emotions = sorted(gemini_emotions.items(), key=lambda x: x[1], reverse=True)
-            gemini_emotions = dict(sorted_emotions[:3])
+            gemini_emotions = dict(sorted_emotions[:3])  # keep top 3
 
             senti_res_new = sentiment_pipeline(context_for_scoring[:512])[0]
             gemini_sentiment = max(senti_res_new, key=lambda x: x["score"])
 
-            # Guardrails on GEMINI output:
-            # If Gemini's suggested rewrite itself is strongly negative, we skip (treat as no rewrite)
+            # Guardrails on Gemini output
             if gemini_sentiment["label"].upper() == "NEGATIVE" and gemini_sentiment["score"] >= 0.8:
-                return {}, f"✅ No rewrite needed. The {level.lower()} reads naturally and clearly."
+                return {}, f"✅ No rewrite needed. The {level.lower()} is clear and well written."
 
             negative_emotions = ["disapproval", "anger", "sadness", "fear", "disgust", "annoyance", "grief", "remorse"]
             for emo, score in gemini_emotions.items():
                 if emo.lower() in negative_emotions and score >= 0.8:
-                    return {}, f"✅ No rewrite needed. The {level.lower()} reads naturally and clearly."
+                    return {}, f"✅ No rewrite needed. The {level.lower()} is clear and well written."
 
-            # If both approval and disapproval are high in the gemini re-score, skip as ambiguous
             if gemini_emotions.get("approval", 0) > 0.6 and gemini_emotions.get("disapproval", 0) > 0.6:
-                return {}, f"✅ No rewrite needed. The {level.lower()} reads naturally and clearly."
+                return {}, f"✅ No rewrite needed. The {level.lower()} is clear and well written."
 
-        # Attach SEO suggestions (lightweight) if possible (but minimal)
-        seo_tips = []
-        try:
-            seo_data = compute_seo_suggestions_enhanced(text, [text])
-            seo_tips = seo_data.get("suggestions", [])[:2]
-        except Exception:
-            seo_tips = []
+        # Badge indicator
+        badge = "✍️"
 
-        # Format the final output: show the Gemini rewrite + its sentiment + top-3 emotions
+        # Format Gemini insight with rewrite emotions & sentiment
         gem_emo_text = ", ".join([f"{k}: {v}" for k, v in gemini_emotions.items()]) if gemini_emotions else "N/A"
         gem_sent_text = f"{gemini_sentiment.get('label','N/A')} ({round(gemini_sentiment.get('score',0),3)})" if gemini_sentiment else "N/A"
 
-        seo_text = ""
-        if seo_tips:
-            seo_text = "\n\n💡 SEO Suggestions:\n- " + "\n- ".join(seo_tips)
-
         final_text = (
-            f"✍️ {response_text}\n\n"
+            f"{badge} {response_text}\n\n"
             f"✨ Gemini Rewrite Sentiment: {gem_sent_text}\n"
             f"✨ Gemini Rewrite Top Emotions: {gem_emo_text}"
-            f"{seo_text}"
         )
         return gemini_emotions, final_text
 
@@ -519,18 +374,6 @@ def analyze_article(headline, paragraphs, lang, emotion_pipeline, sentiment_pipe
         st.write("Headline →", headline)
         st.write("Emotions →", headline_emotions)
         st.write("Sentiment →", headline_sentiment)
-
-        # Show SEO suggestions only once here
-        seo_data = compute_seo_suggestions_enhanced(headline, paragraphs)
-        if seo_data.get("suggestions"):
-            st.markdown("### 💡 SEO Suggestions (headline)")
-            for s in seo_data["suggestions"][:3]:
-                st.write("-", s)
-        if seo_data.get("headline_suggestions"):
-            st.markdown("### 📝 Headline ideas:")
-            for hs in seo_data["headline_suggestions"]:
-                st.write("-", hs)
-
         top3_headline, headline_insight = generate_insight(
             headline, headline_emotions, headline_sentiment, "Headline",
             emotion_pipeline=emotion_pipeline, sentiment_pipeline=sentiment_pipeline
@@ -543,7 +386,7 @@ def analyze_article(headline, paragraphs, lang, emotion_pipeline, sentiment_pipe
         })
 
     # -----------------------
-    # Overall Article Analysis (compute weighted emotions across cleaned paragraphs)
+    # Overall Article Analysis
     # -----------------------
     if paragraphs:
         for p in paragraphs:
@@ -578,17 +421,9 @@ def analyze_article(headline, paragraphs, lang, emotion_pipeline, sentiment_pipe
         })
 
     # -----------------------
-    # Paragraph Analysis (detect sub-headings and avoid SEO spam)
+    # Paragraph Analysis
     # -----------------------
     for p_idx, para in enumerate(paragraphs, start=1):
-        # subheading heuristics
-        is_subheading = (
-            para.strip().endswith("?")
-            or len(para.split()) <= 8
-            or bool(re.match(r"^\d+[\.\)]", para.strip()))
-            or (sum(1 for w in para.split() if w.isupper()) >= 2 and len(para.split()) <= 10)
-        )
-
         para_counter, para_sentiments = Counter(), []
         sentences = split_sentences(para, lang[:2])
         for sentence in sentences:
@@ -603,29 +438,17 @@ def analyze_article(headline, paragraphs, lang, emotion_pipeline, sentiment_pipe
         sorted_para = sorted(para_emotions.items(), key=lambda x: x[1], reverse=True)
         para_emotions = dict(sorted_para[:10])
         para_sentiment = max(para_sentiments, key=lambda x: x["score"]) if para_sentiments else {}
-
-        st.subheader(f"{'🧩 Sub-heading' if is_subheading else '📑 Paragraph'} {p_idx}")
+        st.subheader(f"📑 Paragraph {p_idx}")
         st.write(para)
         st.write("Emotions →", para_emotions)
         st.write("Sentiment →", para_sentiment)
-
-        # Show limited SEO only if NOT a sub-heading and only one focused tip
-        if not is_subheading:
-            try:
-                seo_data = compute_seo_suggestions_enhanced("", [para], top_n_keywords=3)
-                uniq_suggestion = seo_data["suggestions"][0] if seo_data.get("suggestions") else None
-                if uniq_suggestion:
-                    st.markdown(f"💡 SEO Tip: {uniq_suggestion}")
-            except Exception:
-                pass
-
         top3_para, insight = generate_insight(
-            para, para_emotions, para_sentiment, "Sub-heading" if is_subheading else "Paragraph",
+            para, para_emotions, para_sentiment, "Paragraph",
             emotion_pipeline=emotion_pipeline, sentiment_pipeline=sentiment_pipeline
         )
         st.write(insight)
         export_rows.append({
-            "Type": "Sub-heading" if is_subheading else "Paragraph","Text": para,
+            "Type": "Paragraph","Text": para,
             "Emotions": para_emotions,"Sentiment": para_sentiment,
             "Top3": dict(top3_para),"Insight": insight
         })
@@ -644,7 +467,6 @@ text_input = st.text_area("Or paste text here")
 if st.button("🔍 Analyze"):
     with st.spinner("Running analysis... ⏳"):
         if uploaded_file:
-            # streamlit FileUploader returns a BytesIO-like object; docx.Document accepts file-like
             headline, paragraphs = read_and_split_articles(uploaded_file)
         elif url_input.strip():
             headline, paragraphs = read_article_from_url(url_input)
@@ -655,7 +477,6 @@ if st.button("🔍 Analyze"):
         else:
             st.warning("Please provide text input.")
             st.stop()
-
         detected_lang = detect((headline + " " + " ".join(paragraphs))[:200]) if (headline or paragraphs) else "en"
         emotion_pipeline, sentiment_pipeline = load_pipelines(detected_lang)
         export_rows = analyze_article(headline, paragraphs, detected_lang, emotion_pipeline, sentiment_pipeline)
@@ -666,3 +487,4 @@ if st.button("🔍 Analyze"):
             excel_buffer = io.BytesIO()
             df_export.to_excel(excel_buffer, index=False, engine="xlsxwriter")
             st.download_button("⬇️ Download Excel", excel_buffer, "analysis_results.xlsx", "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", use_container_width=True)
+