Computation for Linguists

Working with non-English data

Dr. Andrew M. Byrd

2025-11-10

Plan for the Day

• Review
• Working with Other Languages

Review

• What did you learn last time?

Recap from Last Time

import spacy
nlp_en = spacy.load("en_core_web_sm")

doc = nlp_en("The children ran quickly to their houses.")
for token in doc:
    print(token.text, token.lemma_, token.pos_)

Recap from Last Time

import spacy
from nltk.corpus import wordnet as wn
from nltk.corpus.reader.wordnet import NOUN, VERB, ADJ, ADV

# Load spaCy
nlp = spacy.load("en_core_web_sm")

# Sentence with both noun and verb "bear"
text = "The bears bear their burdens bravely."
doc = nlp(text)

# Map spaCy POS tags to WordNet POS tags -- this is a **function**, we'll get to these soon
def get_wordnet_pos(spacy_pos):
    if spacy_pos.startswith("N"):
        return NOUN
    elif spacy_pos.startswith("V"):
        return VERB
    elif spacy_pos.startswith("J"):
        return ADJ
    elif spacy_pos.startswith("R"):
        return ADV
    return None

# Loop through tokens and look up WordNet entries
for token in doc:
    wn_pos = get_wordnet_pos(token.tag_)
    lemma = token.lemma_.lower()

    if wn_pos and not token.is_stop and not token.is_punct:
        synsets = wn.synsets(lemma, pos=wn_pos)
        print(f"\n{token.text.upper()} ({token.pos_}) → lemma: {lemma}")
        for s in synsets[:3]:  # show just the first 3 senses
            print(f"  - {s.definition()}  [examples: {s.examples()}]")

Semantic Similarity

dog = nlp("dog")[0]._.wordnet.synsets()[0]
comparison = nlp("cat")[0]._.wordnet.synsets()[0]

print("wup:",  dog.wup_similarity(comparison))   # Wu–Palmer similarity

Returning to our verbs of violence

import spacy
from spacy.matcher import PhraseMatcher
from nltk.corpus import wordnet as wn

nlp = spacy.load("en_core_web_sm")

# WordNet helpers (via NLTK) – returns only verb synsets
def verb_senses(word):
    return wn.synsets(word, pos='v')  # verb synsets only

# Collects all (recursive) hyponyms for a verb synset (troponyms in WN terms).
# This function recursively collects all *hyponyms* (a.k.a. “troponyms” for verbs)
# of a given synset. A hyponym is a more *specific* instance of an action.
# Example: the verb “attack.v.01” has hyponyms like “bomb.v.01”, “invade.v.01”, etc.
def all_verb_hyponyms(root):
      seen, stack = set(), [root]
    while stack:
        cur = stack.pop()
        if cur in seen:
            continue
        seen.add(cur)
        # For verbs, .hyponyms() are the troponyms
        stack.extend(cur.hyponyms())
    return {s for s in seen if s.pos() == 'v'}

# This function extracts lemma names (the “canonical” word forms)
# from a set of synsets, e.g. 'strike.v.01' → ['strike', 'hit', 'smite', ...].
# Optionally filters out multiword expressions like "shoot_down".
def lemmas_from_synsets(synsets, keep_multiword=False):
    out = set()
    for s in synsets:
        for lem in s.lemmas():
            name = lem.name().lower()
            if not keep_multiword and "_" in name:
                continue
            out.add(name.replace("_", " "))
    return out

# Build a violence lexicon from WordNet using a few intuitive seeds
seed_verbs = ["attack", "assault", "hit", "strike", "punch", "kick", "stab", "shoot", "beat"]
base_synsets = []
for w in seed_verbs:
    ss = verb_senses(w)
    if ss:
        # take the most “central” sense by picking the one with most hyponyms
        ss_scored = sorted(ss, key=lambda s: len(s.hyponyms()), reverse=True)
        base_synsets.append(ss_scored[0])

# expand via hyponyms (troponyms)
expanded = set()
for s in base_synsets:
    expanded |= all_verb_hyponyms(s)

# collect lemmas (single-token by default)
violent_verb_lemmas = sorted(lemmas_from_synsets(expanded, keep_multiword=False) | set(seed_verbs))
print(f"{len(violent_verb_lemmas)} violent verb lemmas (sample):", violent_verb_lemmas[:25])

# 4) spaCy PhraseMatcher by lemma — IMPORTANT: run full pipeline on patterns
matcher = PhraseMatcher(nlp.vocab, attr="LEMMA")
patterns = list(nlp.pipe(violent_verb_lemmas))  # not make_doc: we need lemmas
matcher.add("VIOLENCE", patterns)

An automated list of words

violent_vs = ['assail', 'assault', 'atom-bomb', 'atomise', 'atomize', 'attack', 'backbite', 'backhand', 'bait', 'bastinado', 'bat', 'batter', 'bayonet', 'beak', 'beat', 'beef', 'beetle', 'beleaguer', 'bellyache', 'bemoan', 'beset', 'besiege', 'best', 'better', 'bewail', 'birdie', 'bitch', 'blast', 'bleat', 'blindside', 'blitz', 'blockade', 'bogey', 'bomb', 'bombard', 'bounce', 'break', 'buffet', 'bulldog', 'bunker', 'bunt', 'bust', 'butt', 'cannon', 'cannonade', 'carom', 'carry', 'charge', 'cheat', 'checkmate', 'chicane', 'chip', 'chop', 'chouse', 'circumvent', 'clap', 'clobber', 'clout', 'coldcock', 'complain', 'connect', 'counterattack', 'counterstrike', 'crab', 'cream', 'croak', 'croquet', 'crump', 'crush', 'cuff', 'dab', 'deck', 'declaim', 'deplore', 'desecrate', 'dishonor', 'dishonour', 'dive-bomb', 'double', 'down', 'dribble', 'drive', 'drub', 'dump', 'dunk', 'eagle', 'ebb', 'eliminate', 'exceed', 'firebomb', 'floor', 'fly', 'foul', 'full', 'gang-rape', 'gas', 'glide-bomb', 'gnarl', 'gripe', 'grizzle', 'grouch', 'ground', 'grouse', 'grumble', 'hammer', 'headbutt', 'heel', 'hen-peck', 'hew', 'hit', 'hole', 'holler', 'hook', 'hydrogen-bomb', 'immobilise', 'immobilize', 'infest', 'invade', 'inveigh', 'jab', 'jockey', 'jump', 'kick', 'kill', 'knap', 'knife', 'knock', 'knuckle', 'kvetch', 'lament', 'lash', 'lick', 'loft', 'master', 'mate', 'molest', 'murmur', 'mutter', 'nag', 'nuke', 'occupy', 'out-herod', 'outbrave', 'outcry', 'outdo', 'outdraw', 'outfight', 'outflank', 'outfox', 'outgeneral', 'outgo', 'outgrow', 'outmaneuver', 'outmanoeuvre', 'outmarch', 'outmatch', 'outpace', 'outperform', 'outplay', 'outpoint', 'outrage', 'outrange', 'outroar', 'outsail', 'outscore', 'outsell', 'outshine', 'outshout', 'outsmart', 'outstrip', 'outwear', 'outweigh', 'outwit', 'overcome', 'overmaster', 'overpower', 'overreach', 'overrun', 'overwhelm', 'paste', 'pat', 'pattern-bomb', 'peck', 'pelt', 'pepper', 'percuss', 'pick', 'pip', 'pitch', 'plain', 'play', 'plug', 'poniard', 'pop', 'profane', 'protest', 'pull', 'punch', 'putt', 'quetch', 'racket', 'raid', 'rail', 'rap', 'rape', 'ravish', 'reassail', 'repine', 'report', 'retaliate', 'rout', 'rush', 'savage', 'sclaff', 'scold', 'scoop', 'screw', 'set', 'shaft', 'shame', 'shank', 'shell', 'shoot', 'sic', 'sideswipe', 'single', 'skip-bomb', 'slam-dunk', 'slap', 'sledge', 'sledgehammer', 'slice', 'smash', 'snag', 'snap', 'snick', 'spread-eagle', 'spreadeagle', 'spur', 'squawk', 'stab', 'steamroll', 'steamroller', 'storm', 'strafe', 'strike', 'stroke', 'subdue', 'submarine', 'surmount', 'surpass', 'surprise', 'surround', 'tap', 'teargas', 'thrash', 'thresh', 'tip', 'toe', 'top', 'torpedo', 'triple', 'trounce', 'trump', 'undercut', 'upstage', 'urticate', 'vanquish', 'violate', 'volley', 'whang', 'whine', 'whip', 'whomp', 'worst', 'yammer', 'yawp', 'zap']

Rule-Based Matching (verbs of violence, e.g.)

• Copy the list on the previous slide.
• Then return to the code from before. Add a couple additional violent verbs to the VP to see if the list is comprehensive enough.

import spacy
from spacy.matcher import PhraseMatcher

nlp = spacy.load("en_core_web_sm")

# Paste the list here.

patterns = [nlp(v) for v in violent_vs]
matcher = PhraseMatcher(nlp.vocab, attr="LEMMA")
doc = nlp("They punched, kicked, and attacked the intruder before fleeing.")
matcher.add("VIOLENCE", patterns)
[(doc[s:e].text, doc[s:e].lemma_) for _, s, e in matcher(doc)]

Working with Other Languages

spaCy Multilingual Models

spaCy provides language-specific pipelines trained for each language.

Language	Model Name	Example
English	en_core_web_sm	“The students studied hard.”
French	fr_core_news_sm	“Les étudiants ont étudié dur.”
Spanish	es_core_news_sm	“Los estudiantes estudiaron mucho.”
German	de_core_news_sm	“Die Studenten haben fleißig gelernt.”

Download additional spaCy models

• Let’s download spaCy models for Spanish, German, and French

python -m spacy download es_core_news_sm
python -m spacy download de_core_news_sm
python -m spacy download fr_core_news_sm

Tokenization in Multiple Languages

import spacy

nlp_en = spacy.load("en_core_web_sm")
nlp_fr = spacy.load("fr_core_news_sm")
nlp_es = spacy.load("es_core_news_sm")
nlp_de = spacy.load("de_core_news_sm")

text_en = "The students have analyzed Proto-Indo-European roots."
text_fr = "Les étudiants ont analysé les racines proto-indo-européennes."
text_es = "Los estudiantes analizaron las raíces protoindoeuropeas."
text_de = "Die Studenten analysierten die indogermanischen Wurzeln."

for lang, nlp, text in [("English", nlp_en, text_en), ("French", nlp_fr, text_fr), ("Spanish", nlp_es, text_es), ("German", nlp_de, text_de)]:
    doc = nlp(text)
    print(f"\n {lang} Tokens:")
    print([t.text for t in doc])

Activity: Tokenization

• Write a new sentence in English.
• Translate that sentence into French, Spanish, and German (you may use Google Translate)
• Tokenize that sentence in all 4 languages.

Lemmatization and POS Across Languages

for lang, nlp, text in [("English", nlp_en, text_en), ("French", nlp_fr, text_fr), ("Spanish", nlp_es, text_es), ("German", nlp_de, text_de)]:
    doc = nlp(text)
    print(f"\n {lang} Lemmas:")
    for token in doc:
        print(token.text, "→", token.lemma_, token.pos_)

Activity: Lemmatization

• Lemmatize and identify POS for your newly created sentence in English, French, Spanish, and German.

Creating Cross-Language DataFrames

import pandas as pd

def lemma_table(nlp, text, language):
    doc = nlp(text)
    data = [(t.text, t.lemma_, t.pos_) for t in doc if t.is_alpha]
    return pd.DataFrame(data, columns=["word", "lemma", "pos"]).assign(language=language)

df_all = pd.concat([
    lemma_table(nlp_en, text_en, "English"),
    lemma_table(nlp_fr, text_fr, "French"),
    lemma_table(nlp_es, text_es, "Spanish"),
    lemma_table(nlp_de, text_de, "German")
])

df_all

Activity: Pandas

• Create a df for your sentence, as done above.

Frequency Comparisons Across Languages

• Let’s use our Counter function to compare how common certain words are.
• Before we begin, let’s load up two texts in English & Spanish.

text_en = "Whereas recognition of the inherent dignity and of the equal and inalienable rights of all members of the human family is the foundation of freedom, justice and peace in the world, the peoples of the United Nations have reaffirmed their faith in fundamental human rights and in the dignity and worth of the human person. They have resolved to promote social progress and better standards of life in larger freedom."

text_es = "Considerando que el reconocimiento de la dignidad intrínseca y de los derechos iguales e inalienables de todos los miembros de la familia humana constituye la base de la libertad, la justicia y la paz en el mundo, los pueblos de las Naciones Unidas han reafirmado su fe en los derechos humanos fundamentales y en la dignidad y el valor de la persona humana. Han decidido promover el progreso social y elevar el nivel de vida dentro de una libertad más amplia."

Frequency Comparisons

from collections import Counter
import spacy

nlp_en = spacy.load("en_core_web_sm")
nlp_es = spacy.load("es_core_news_sm")

def is_def_det(tok, include_contractions=False, lang="en"):
# True for tokens that are definite determiners
  if tok.pos_ == "DET" and "Def" in tok.morph.get("Definite"):
    return True
  if include_contractions and lang == "es" and tok.text.lower() in {"al", "del"}:
    return True
    return False

def def_det_stats(nlp, text, lang_label, include_contractions=False):
  doc = nlp(text)
  # word-like tokens only for the denominator
  word_tokens = [t for t in doc if t.is_alpha]
  total = len(word_tokens)
  def_dets = [t.text for t in doc if is_def_det(t, include_contractions, lang=lang_label[:2].lower())]
  n = len(def_dets)
  rate = (n / total) if total else 0.0
  print(f"{lang_label}: {n} definite determiners / {total} word tokens ({rate:.2%})")
  print("Top forms:", Counter(w.lower() for w in def_dets).most_common())

# Run (set include_contractions=True if you want to count 'al'/'del' in Spanish)
def_det_stats(nlp_en, text_en, "English", include_contractions=False)
def_det_stats(nlp_es, text_es, "Spanish", include_contractions=False)

Activity:

• Copy the below texts and calculate how many determiners are in German & French.

text_fr = "Considérant que la reconnaissance de la dignité inhérente et des droits égaux et inaliénables de tous les membres de la famille humaine constitue le fondement de la liberté, de la justice et de la paix dans le monde, les peuples des Nations Unies ont réaffirmé leur foi dans les droits fondamentaux de l’homme, dans la dignité et la valeur de la personne humaine. Ils se sont engagés à favoriser le progrès social et à élever le niveau de vie dans une liberté plus grande."

text_de = "Da die Anerkennung der angeborenen Würde und der gleichen und unveräußerlichen Rechte aller Mitglieder der menschlichen Familie die Grundlage der Freiheit, der Gerechtigkeit und des Friedens in der Welt bildet, haben die Völker der Vereinten Nationen ihren Glauben an die grundlegenden Menschenrechte sowie an die Würde und den Wert der menschlichen Person erneut bekräftigt. Sie haben beschlossen, sozialen Fortschritt zu fördern und den Lebensstandard in größerer Freiheit zu erhöhen."

nlp_fr = spacy.load("fr_core_news_sm")
nlp_de = spacy.load("de_core_news_sm")

def_det_stats(nlp_fr, text_fr, "French", include_contractions=False)
def_det_stats(nlp_de, text_de, "German", include_contractions=False)

WordNet in Other Languages

• WordNet itself is English-centric, but Open Multilingual Wordnet (OMW) links many languages to the Princeton WordNet synsets.
• Try:
  • Note: qcn == Simplified Chinese

# pip install pycountry
import pycountry
from nltk.corpus import wordnet as wn
# List languages available via OMW (IDs)
codes = sorted(wn.langs())

for code in codes:
    lang = pycountry.languages.get(alpha_3=code)
    if lang:
        print(f"{code} → {lang.name}")
    else:
        print(f"{code} → (not found)")

Looking up lemmas by code

# Example: Spanish lemmas linked to the English synset for 'dog.n.01'
ss = wn.synset('dog.n.01')
[lem.name() for lem in ss.lemmas(lang='spa')]

Activity

• Using the 3-letter ISO code and the above code, look up the words meaning dog (n.01) in French

# Example: Spanish lemmas linked to the English synset for 'dog.n.01'
ss = wn.synset('dog.n.01')
[lem.name() for lem in ss.lemmas(lang='fra')]

• Wait, where’s German?