def greedy_generate(model, start_word, length=15):
    """Always pick the most likely next word"""
    current = start_word
    output = [current]
    
    for _ in range(length - 1):
        if current not in model:
            break
        next_counts = model[current]
        # Pick the word with highest count
        best_word = max(next_counts, key=next_counts.get)
        output.append(best_word)
        current = best_word
        
        if best_word in ['.', '<END>']:
            break
    
    return ' '.join(output)

import heapq
from collections import defaultdict, Counter

def beam_search(model, start_word, beam_width=3, max_length=10):
    """
    Beam search text generation.
    model: dict of {word: Counter of next words}
    """
    import math
    
    # Beam: list of (log_prob, word_sequence)
    beam = [(0.0, [start_word])]
    completed = []
    
    for _ in range(max_length):
        candidates = []
        
        for log_prob, sequence in beam:
            current_word = sequence[-1]
            
            if current_word == '<END>' or current_word == '.':
                completed.append((log_prob, sequence))
                continue
            
            if current_word not in model:
                completed.append((log_prob, sequence))
                continue
            
            total = sum(model[current_word].values())
            
            # Expand: try every possible next word
            for next_word, count in model[current_word].items():
                prob = count / total
                new_log_prob = log_prob + math.log(prob + 1e-10)
                candidates.append((new_log_prob, sequence + [next_word]))
        
        if not candidates:
            break
        
        # Keep only top beam_width candidates
        beam = heapq.nlargest(beam_width, candidates, key=lambda x: x[0])
    
    completed.extend(beam)
    
    # Return the best sequence
    best_log_prob, best_sequence = max(completed, key=lambda x: x[0])
    return ' '.join(best_sequence), best_log_prob

# Example usage
from collections import defaultdict, Counter
import random

# Build a small bigram model
corpus = """
i love cats and dogs . 
i love pizza and pasta .
cats love fish and milk .
dogs love bones and running .
the cat sat on the mat .
the dog ran in the park .
"""

model = defaultdict(Counter)
words = corpus.split()
for i in range(len(words) - 1):
    model[words[i]][words[i+1]] += 1

# Generate with beam search
result, score = beam_search(model, "i", beam_width=3, max_length=8)
print(f"Beam search output: {result}")
print(f"Log probability: {score:.2f}")

import numpy as np

def temperature_sample(word_counts, temperature=1.0):
    """
    Sample next word with temperature control.
    """
    words = list(word_counts.keys())
    counts = np.array(list(word_counts.values()), dtype=float)
    
    # Apply temperature: divide log probabilities by T
    scaled = counts ** (1.0 / temperature)
    probs = scaled / scaled.sum()
    
    return np.random.choice(words, p=probs)

# Compare temperatures on "love" → next word
from collections import Counter
love_follows = Counter({'cats': 10, 'dogs': 8, 'pizza': 3, 'running': 1, 'jazz': 1})

print("Next word after 'love':")
print("\nT=0.3 (conservative, repeats top words):")
for _ in range(5):
    print(" ", temperature_sample(love_follows, temperature=0.3))

print("\nT=1.0 (normal sampling):")
for _ in range(5):
    print(" ", temperature_sample(love_follows, temperature=1.0))

print("\nT=2.0 (adventurous, rare words appear more):")
for _ in range(5):
    print(" ", temperature_sample(love_follows, temperature=2.0))

from sklearn.feature_extraction.text import TfidfVectorizer
import numpy as np

def tfidf_summarise(text, num_sentences=3):
    """Extractive summarisation using TF-IDF sentence scoring"""
    
    # Split into sentences
    import re
    sentences = re.split(r'(?<=[.!?])\s+', text.strip())
    
    if len(sentences) <= num_sentences:
        return text
    
    # Build TF-IDF over sentences (treating each sentence as a document)
    vectorizer = TfidfVectorizer(stop_words='english')
    tfidf_matrix = vectorizer.fit_transform(sentences)
    
    # Score each sentence: sum of its word TF-IDF scores
    sentence_scores = np.asarray(tfidf_matrix.sum(axis=1)).flatten()
    
    # Pick top N sentences (preserve original order)
    top_indices = sorted(
        np.argsort(sentence_scores)[-num_sentences:].tolist()
    )
    
    summary = ' '.join(sentences[i] for i in top_indices)
    return summary

# Test
article = """
The Amazon rainforest is the world's largest tropical rainforest, covering over 
five and a half million square kilometres. It is home to an estimated 10% of all 
species on Earth, including jaguars, anacondas, and hundreds of bird species. 
The forest plays a critical role in regulating the global climate by absorbing 
enormous amounts of carbon dioxide. Deforestation is the biggest threat to the 
Amazon, with cattle ranching and agriculture responsible for most land clearance. 
In recent years, the rate of deforestation has accelerated, alarming scientists 
and conservationists worldwide. Protecting the Amazon is considered essential 
for meeting global climate targets and preserving biodiversity. Several 
international agreements aim to reduce deforestation, but enforcement remains 
a significant challenge.
"""

summary = tfidf_summarise(article, num_sentences=3)
print("SUMMARY:")
print(summary)

def position_score(sentence_index, total_sentences):
    """
    Sentences at the beginning score higher.
    Simple linear decay.
    """
    return 1.0 - (sentence_index / total_sentences) * 0.5

from collections import Counter
from nltk.corpus import stopwords
from nltk.tokenize import word_tokenize

def keyword_score(sentence, document_word_freq, stop_words):
    """Score a sentence by the frequency of its content words"""
    words = word_tokenize(sentence.lower())
    content_words = [w for w in words if w.isalpha() and w not in stop_words]
    
    if not content_words:
        return 0
    
    return sum(document_word_freq.get(w, 0) for w in content_words) / len(content_words)

# pip install sumy  (implements TextRank and other summarisers)
from sumy.parsers.plaintext import PlaintextParser
from sumy.nlp.tokenizers import Tokenizer
from sumy.summarizers.text_rank import TextRankSummarizer

def textrank_summarise(text, num_sentences=3):
    parser = PlaintextParser.from_string(text, Tokenizer("english"))
    summarizer = TextRankSummarizer()
    summary = summarizer(parser.document, num_sentences)
    return ' '.join(str(sentence) for sentence in summary)

# Or using the popular gensim implementation
from gensim.summarization import summarize  # gensim < 4.0

summary = summarize(article, ratio=0.3)  # Keep 30% of original

# Simplified illustration of phrase-based translation lookup
phrase_table = {
    "the cat": "el gato",
    "the dog": "el perro",
    "is running": "está corriendo",
    "in the park": "en el parque",
    "the cat is running": "el gato está corriendo",
    "in the garden": "en el jardín"
}

def simple_phrase_translate(sentence, phrase_table):
    """
    Very simplified phrase-based translation.
    Tries to match longest phrase first (greedy).
    """
    words = sentence.lower().split()
    result = []
    i = 0
    
    while i < len(words):
        # Try to match longest phrase starting at position i
        matched = False
        for length in range(len(words) - i, 0, -1):
            phrase = ' '.join(words[i:i+length])
            if phrase in phrase_table:
                result.append(phrase_table[phrase])
                i += length
                matched = True
                break
        
        if not matched:
            result.append(f"[{words[i]}?]")  # unknown word
            i += 1
    
    return ' '.join(result)

print(simple_phrase_translate("the cat is running in the park", phrase_table))
# "el gato está corriendo en el parque"

from nltk.translate.bleu_score import sentence_bleu, SmoothingFunction

reference = [["the", "cat", "is", "on", "the", "mat"]]
candidate1 = ["the", "cat", "is", "on", "the", "mat"]
candidate2 = ["the", "cat", "sat", "on", "the", "mat"]
candidate3 = ["a", "feline", "rests", "upon", "the", "rug"]

smooth = SmoothingFunction().method1

print(f"Perfect match BLEU:  {sentence_bleu(reference, candidate1, smoothing_function=smooth):.3f}")
print(f"One word off BLEU:   {sentence_bleu(reference, candidate2, smoothing_function=smooth):.3f}")
print(f"Different vocab BLEU:{sentence_bleu(reference, candidate3, smoothing_function=smooth):.3f}")