Text Analysis with large language models
Emi Tanaka
4th June 2025
These slides are best viewed on a modern browser like Google Chrome on a desktop or laptop. You can navigate the slides using the left and right arrows.
library(stringr)
text <- c("This is a sample text with some numbers 12345 and
special characters !@#$%^&*()",
"Another example with extra spaces and punctuation...!!!")
text |>
str_to_lower() |>
str_remove_all("[^[:alnum:][:space:]]") |>
str_squish()[1] "this is a sample text with some numbers 12345 and special characters"
[2] "another example with extra spaces and punctuation" SnowballC and textstem packages [1] "a" "a's" "able" "about"
[5] "above" "according" "accordingly" "across"
[9] "actually" "after" "afterwards" "again"
[13] "against" "ain't" "all" "allow"
[17] "allows" "almost" "alone" "along"
[21] "already" "also" "although" "always"
[25] "am" "among" "amongst" "an"
[29] "and" "another" "any" "anybody"
[33] "anyhow" "anyone" "anything" "anyway"
[37] "anyways" "anywhere" "apart" "appear"
[41] "appreciate" "appropriate" "are" "aren't"
[45] "around" "as" "aside" "ask"
[49] "asking" "associated" "at" "available"
[53] "away" "awfully" "b" "be"
[57] "became" "because" "become" "becomes"
[61] "becoming" "been" "before" "beforehand"
[65] "behind" "being" "believe" "below"
[69] "beside" "besides" "best" "better"
[73] "between" "beyond" "both" "brief"
[77] "but" "by" "c" "c'mon"
[81] "c's" "came" "can" "can't"
[85] "cannot" "cant" "cause" "causes"
[89] "certain" "certainly" "changes" "clearly"
[93] "co" "com" "come" "comes"
[97] "concerning" "consequently" "consider" "considering"
[101] "contain" "containing" "contains" "corresponding"
[105] "could" "couldn't" "course" "currently"
[109] "d" "definitely" "described" "despite"
[113] "did" "didn't" "different" "do"
[117] "does" "doesn't" "doing" "don't"
[121] "done" "down" "downwards" "during"
[125] "e" "each" "edu" "eg"
[129] "eight" "either" "else" "elsewhere"
[133] "enough" "entirely" "especially" "et"
[137] "etc" "even" "ever" "every"
[141] "everybody" "everyone" "everything" "everywhere"
[145] "ex" "exactly" "example" "except"
[149] "f" "far" "few" "fifth"
[153] "first" "five" "followed" "following"
[157] "follows" "for" "former" "formerly"
[161] "forth" "four" "from" "further"
[165] "furthermore" "g" "get" "gets"
[169] "getting" "given" "gives" "go"
[173] "goes" "going" "gone" "got"
[177] "gotten" "greetings" "h" "had"
[181] "hadn't" "happens" "hardly" "has"
[185] "hasn't" "have" "haven't" "having"
[189] "he" "he's" "hello" "help"
[193] "hence" "her" "here" "here's"
[197] "hereafter" "hereby" "herein" "hereupon"
[201] "hers" "herself" "hi" "him"
[205] "himself" "his" "hither" "hopefully"
[209] "how" "howbeit" "however" "i"
[213] "i'd" "i'll" "i'm" "i've"
[217] "ie" "if" "ignored" "immediate"
[221] "in" "inasmuch" "inc" "indeed"
[225] "indicate" "indicated" "indicates" "inner"
[229] "insofar" "instead" "into" "inward"
[233] "is" "isn't" "it" "it'd"
[237] "it'll" "it's" "its" "itself"
[241] "j" "just" "k" "keep"
[245] "keeps" "kept" "know" "knows"
[249] "known" "l" "last" "lately"
[253] "later" "latter" "latterly" "least"
[257] "less" "lest" "let" "let's"
[261] "like" "liked" "likely" "little"
[265] "look" "looking" "looks" "ltd"
[269] "m" "mainly" "many" "may"
[273] "maybe" "me" "mean" "meanwhile"
[277] "merely" "might" "more" "moreover"
[281] "most" "mostly" "much" "must"
[285] "my" "myself" "n" "name"
[289] "namely" "nd" "near" "nearly"
[293] "necessary" "need" "needs" "neither"
[297] "never" "nevertheless" "new" "next"
[301] "nine" "no" "nobody" "non"
[305] "none" "noone" "nor" "normally"
[309] "not" "nothing" "novel" "now"
[313] "nowhere" "o" "obviously" "of"
[317] "off" "often" "oh" "ok"
[321] "okay" "old" "on" "once"
[325] "one" "ones" "only" "onto"
[329] "or" "other" "others" "otherwise"
[333] "ought" "our" "ours" "ourselves"
[337] "out" "outside" "over" "overall"
[341] "own" "p" "particular" "particularly"
[345] "per" "perhaps" "placed" "please"
[349] "plus" "possible" "presumably" "probably"
[353] "provides" "q" "que" "quite"
[357] "qv" "r" "rather" "rd"
[361] "re" "really" "reasonably" "regarding"
[365] "regardless" "regards" "relatively" "respectively"
[369] "right" "s" "said" "same"
[373] "saw" "say" "saying" "says"
[377] "second" "secondly" "see" "seeing"
[381] "seem" "seemed" "seeming" "seems"
[385] "seen" "self" "selves" "sensible"
[389] "sent" "serious" "seriously" "seven"
[393] "several" "shall" "she" "should"
[397] "shouldn't" "since" "six" "so"
[401] "some" "somebody" "somehow" "someone"
[405] "something" "sometime" "sometimes" "somewhat"
[409] "somewhere" "soon" "sorry" "specified"
[413] "specify" "specifying" "still" "sub"
[417] "such" "sup" "sure" "t"
[421] "t's" "take" "taken" "tell"
[425] "tends" "th" "than" "thank"
[429] "thanks" "thanx" "that" "that's"
[433] "thats" "the" "their" "theirs"
[437] "them" "themselves" "then" "thence"
[441] "there" "there's" "thereafter" "thereby"
[445] "therefore" "therein" "theres" "thereupon"
[449] "these" "they" "they'd" "they'll"
[453] "they're" "they've" "think" "third"
[457] "this" "thorough" "thoroughly" "those"
[461] "though" "three" "through" "throughout"
[465] "thru" "thus" "to" "together"
[469] "too" "took" "toward" "towards"
[473] "tried" "tries" "truly" "try"
[477] "trying" "twice" "two" "u"
[481] "un" "under" "unfortunately" "unless"
[485] "unlikely" "until" "unto" "up"
[489] "upon" "us" "use" "used"
[493] "useful" "uses" "using" "usually"
[497] "uucp" "v" "value" "various"
[501] "very" "via" "viz" "vs"
[505] "w" "want" "wants" "was"
[509] "wasn't" "way" "we" "we'd"
[513] "we'll" "we're" "we've" "welcome"
[517] "well" "went" "were" "weren't"
[521] "what" "what's" "whatever" "when"
[525] "whence" "whenever" "where" "where's"
[529] "whereafter" "whereas" "whereby" "wherein"
[533] "whereupon" "wherever" "whether" "which"
[537] "while" "whither" "who" "who's"
[541] "whoever" "whole" "whom" "whose"
[545] "why" "will" "willing" "wish"
[549] "with" "within" "without" "won't"
[553] "wonder" "would" "would" "wouldn't"
[557] "x" "y" "yes" "yet"
[561] "you" "you'd" "you'll" "you're"
[565] "you've" "your" "yours" "yourself"
[569] "yourselves" "z" "zero" "i"
[573] "me" "my" "myself" "we"
[577] "our" "ours" "ourselves" "you"
[581] "your" "yours" "yourself" "yourselves"
[585] "he" "him" "his" "himself"
[589] "she" "her" "hers" "herself"
[593] "it" "its" "itself" "they"
[597] "them" "their" "theirs" "themselves"
[601] "what" "which" "who" "whom"
[605] "this" "that" "these" "those"
[609] "am" "is" "are" "was"
[613] "were" "be" "been" "being"
[617] "have" "has" "had" "having"
[621] "do" "does" "did" "doing"
[625] "would" "should" "could" "ought"
[629] "i'm" "you're" "he's" "she's"
[633] "it's" "we're" "they're" "i've"
[637] "you've" "we've" "they've" "i'd"
[641] "you'd" "he'd" "she'd" "we'd"
[645] "they'd" "i'll" "you'll" "he'll"
[649] "she'll" "we'll" "they'll" "isn't"
[653] "aren't" "wasn't" "weren't" "hasn't"
[657] "haven't" "hadn't" "doesn't" "don't"
[661] "didn't" "won't" "wouldn't" "shan't"
[665] "shouldn't" "can't" "cannot" "couldn't"
[669] "mustn't" "let's" "that's" "who's"
[673] "what's" "here's" "there's" "when's"
[677] "where's" "why's" "how's" "a"
[681] "an" "the" "and" "but"
[685] "if" "or" "because" "as"
[689] "until" "while" "of" "at"
[693] "by" "for" "with" "about"
[697] "against" "between" "into" "through"
[701] "during" "before" "after" "above"
[705] "below" "to" "from" "up"
[709] "down" "in" "out" "on"
[713] "off" "over" "under" "again"
[717] "further" "then" "once" "here"
[721] "there" "when" "where" "why"
[725] "how" "all" "any" "both"
[729] "each" "few" "more" "most"
[733] "other" "some" "such" "no"
[737] "nor" "not" "only" "own"
[741] "same" "so" "than" "too"
[745] "very" "a" "about" "above"
[749] "across" "after" "again" "against"
[753] "all" "almost" "alone" "along"
[757] "already" "also" "although" "always"
[761] "among" "an" "and" "another"
[765] "any" "anybody" "anyone" "anything"
[769] "anywhere" "are" "area" "areas"
[773] "around" "as" "ask" "asked"
[777] "asking" "asks" "at" "away"
[781] "back" "backed" "backing" "backs"
[785] "be" "became" "because" "become"
[789] "becomes" "been" "before" "began"
[793] "behind" "being" "beings" "best"
[797] "better" "between" "big" "both"
[801] "but" "by" "came" "can"
[805] "cannot" "case" "cases" "certain"
[809] "certainly" "clear" "clearly" "come"
[813] "could" "did" "differ" "different"
[817] "differently" "do" "does" "done"
[821] "down" "down" "downed" "downing"
[825] "downs" "during" "each" "early"
[829] "either" "end" "ended" "ending"
[833] "ends" "enough" "even" "evenly"
[837] "ever" "every" "everybody" "everyone"
[841] "everything" "everywhere" "face" "faces"
[845] "fact" "facts" "far" "felt"
[849] "few" "find" "finds" "first"
[853] "for" "four" "from" "full"
[857] "fully" "further" "furthered" "furthering"
[861] "furthers" "gave" "general" "generally"
[865] "get" "gets" "give" "given"
[869] "gives" "go" "going" "good"
[873] "goods" "got" "great" "greater"
[877] "greatest" "group" "grouped" "grouping"
[881] "groups" "had" "has" "have"
[885] "having" "he" "her" "here"
[889] "herself" "high" "high" "high"
[893] "higher" "highest" "him" "himself"
[897] "his" "how" "however" "i"
[901] "if" "important" "in" "interest"
[905] "interested" "interesting" "interests" "into"
[909] "is" "it" "its" "itself"
[913] "just" "keep" "keeps" "kind"
[917] "knew" "know" "known" "knows"
[921] "large" "largely" "last" "later"
[925] "latest" "least" "less" "let"
[929] "lets" "like" "likely" "long"
[933] "longer" "longest" "made" "make"
[937] "making" "man" "many" "may"
[941] "me" "member" "members" "men"
[945] "might" "more" "most" "mostly"
[949] "mr" "mrs" "much" "must"
[953] "my" "myself" "necessary" "need"
[957] "needed" "needing" "needs" "never"
[961] "new" "new" "newer" "newest"
[965] "next" "no" "nobody" "non"
[969] "noone" "not" "nothing" "now"
[973] "nowhere" "number" "numbers" "of"
[977] "off" "often" "old" "older"
[981] "oldest" "on" "once" "one"
[985] "only" "open" "opened" "opening"
[989] "opens" "or" "order" "ordered"
[993] "ordering" "orders" "other" "others"
[997] "our" "out" "over" "part"
[1001] "parted" "parting" "parts" "per"
[1005] "perhaps" "place" "places" "point"
[1009] "pointed" "pointing" "points" "possible"
[1013] "present" "presented" "presenting" "presents"
[1017] "problem" "problems" "put" "puts"
[1021] "quite" "rather" "really" "right"
[1025] "right" "room" "rooms" "said"
[1029] "same" "saw" "say" "says"
[1033] "second" "seconds" "see" "seem"
[1037] "seemed" "seeming" "seems" "sees"
[1041] "several" "shall" "she" "should"
[1045] "show" "showed" "showing" "shows"
[1049] "side" "sides" "since" "small"
[1053] "smaller" "smallest" "some" "somebody"
[1057] "someone" "something" "somewhere" "state"
[1061] "states" "still" "still" "such"
[1065] "sure" "take" "taken" "than"
[1069] "that" "the" "their" "them"
[1073] "then" "there" "therefore" "these"
[1077] "they" "thing" "things" "think"
[1081] "thinks" "this" "those" "though"
[1085] "thought" "thoughts" "three" "through"
[1089] "thus" "to" "today" "together"
[1093] "too" "took" "toward" "turn"
[1097] "turned" "turning" "turns" "two"
[1101] "under" "until" "up" "upon"
[1105] "us" "use" "used" "uses"
[1109] "very" "want" "wanted" "wanting"
[1113] "wants" "was" "way" "ways"
[1117] "we" "well" "wells" "went"
[1121] "were" "what" "when" "where"
[1125] "whether" "which" "while" "who"
[1129] "whole" "whose" "why" "will"
[1133] "with" "within" "without" "work"
[1137] "worked" "working" "works" "would"
[1141] "year" "years" "yet" "you"
[1145] "young" "younger" "youngest" "your"
[1149] "yours" library(tidytext)
library(tidyverse)
words <- data.frame(
text = c(
"Once upon a time, in the Kingdom of Probabilia, lived Princess Percentia.",
"She wasn't like the other princesses, who dreamed of dancing and dashing princes.",
"Percentia adored numbers. While her sisters embroidered samplers,
Percentia meticulously collected data on the local butterfly population,
categorized cloud formations, and charted the daily rainfall."
)) |>
unnest_tokens(word, text, token = "words")
words word
1 once
2 upon
3 a
4 time
5 in
6 the
7 kingdom
8 of
9 probabilia
10 lived
11 princess
12 percentia
13 she
14 wasn't
15 like
16 the
17 other
18 princesses
19 who
20 dreamed
21 of
22 dancing
23 and
24 dashing
25 princes
26 percentia
27 adored
28 numbers
29 while
30 her
31 sisters
32 embroidered
33 samplers
34 percentia
35 meticulously
36 collected
37 data
38 on
39 the
40 local
41 butterfly
42 population
43 categorized
44 cloud
45 formations
46 and
47 charted
48 the
49 daily
50 rainfall word n
1 the 4
2 percentia 3
3 and 2
4 of 2
5 a 1
6 adored 1
7 butterfly 1
8 categorized 1
9 charted 1
10 cloud 1
11 collected 1
12 daily 1
13 dancing 1
14 dashing 1
15 data 1
16 dreamed 1
17 embroidered 1
18 formations 1
19 her 1
20 in 1
21 kingdom 1
22 like 1
23 lived 1
24 local 1
25 meticulously 1
26 numbers 1
27 on 1
28 once 1
29 other 1
30 population 1
31 princes 1
32 princess 1
33 princesses 1
34 probabilia 1
35 rainfall 1
36 samplers 1
37 she 1
38 sisters 1
39 time 1
40 upon 1
41 wasn't 1
42 while 1
43 who 1emitanaka.org/workshop-text-analysis-2025/