1

 Week 1: Write a python program to perform tokenization by word and

sentence using nltk.

Program for sentence tokenization:

import nltk

nltk.download('punkt') # Download the necessary tokenization models

from nltk.tokenize import sent_tokenize

def tokenize_sentences(text):

sentences = sent_tokenize(text)

return sentences

text = "NLTK is a leading platform for building Python programs to work with human language

data. It provides easy-to-use interfaces to over 50 corpora and lexical resources such as WordNet,

along with a suite of text processing libraries for classification, tokenization, stemming, tagging,

parsing, and semantic reasoning, wrappers for industrial-strength NLP libraries, and an active

discussion forum."

sentences = tokenize_sentences(text)

for i, sentence in enumerate(sentences):

print(f"Sentence {i+1}: {sentence}")

import nltk

from nltk.tokenize import word_tokenize

word_tokenize('won’t')

:--Program for word Tokenization:

import nltk

nltk.download('punkt') # Download the necessary tokenization models

from nltk.tokenize import word_tokenize

def tokenize_words(text):

words = word_tokenize(text)

return words

text = "NLTK is a leading platform for building Python programs to work with human language

data."

words = tokenize_words(text)

print(words)

Comments

Post a Comment

Popular posts from this blog

Web

                         STRING   <?php $mystr="php is a server side scripting language"; echo "<strong> the given string is :</strong> $mystr <br>"; echo "<strong> the length of string is :</strong>".strlen($mystr)."<br>"; echo "<strong> the count of string is :</strong>".str_word_count($mystr)."<br>"; echo "<strong> the reverse of string is :</strong>".strrev($mystr)."<br>"; $substr="side"; echo "<strong> the position of string is '$substr' in the string is :</strong>".strpos($mystr,$substr);                     Files <?php $fname="file1.txt"; $fptr=fopen($fname,"r"); $content=fread($fptr,filesize($fname)); $fptw=fopen("file2.txt","w"); fwrite($fptw,$content); echo"<strong>the content copied in file2 is:</strong> <br>".$c...
Read more

Lab 1 ai

 Dfs.pl s[a,b). s(a,c). s(b,d). s(b,e). s(c,f). s(c,g). s(d,h). s(e,i). s(e,j). s(f,k). goal(f). goal(j). solve(Start,Solution):- breadthfirst([[Start]],Solution). breadthfirst([[Node | Path] ].[Node|Path]]:- goal(Node). breadthfirst([Path | Paths] Solution):- extend(Path,NewPaths), write(NewPaths), nl, conc(Paths,NewPaths, Paths1), breadthfirst(Paths1,Solution). extend([NodePath],NewPaths):- bagof([NewNode, Node | Path),(s(Node, NewNode),not(member(NewNode, [Node | Path]))),NewPaths),!. extend([]). conc([],L,L). Scanned with OKEN Scanner bfs.pl s(a,b). s(a,c). s(b,d). s(b,e). s(c,f). s(c,g). / s(d,h). s(e,i). s(e,j). s(f,k). goal(f). goal(j). member(X,[X|_]). member(X,_|Tail]):- member(X,Tail). solve(Node,Solution):- depthfirst([],Node,Solution). depthfirst(Path, Node,[Node | Path]):- goal(Node). depthfirst(Path,Node,Sol):- s(Node,Node1), not(member(Node1,Path)), depthfirst([Node | Path], Node1,Sol). Scanned with OKEN Scanner conc([X/L1],L2,[X|L3]):- write('conc'), write(X), w...
Read more
 def aStarAlgo(start_node, stop_node): open_set = set(start_node) closed_set = set() g={} parents = {} g[start_node] = 0 parents[start_node] = start_node while len(open_set) > 0: n = None for v in open_set: if n == None or g[v] + heuristic(v) <g[n] + heuristic(n): n=v if n stop_node or Graph_nodes[n] == None: pass else: for (m, weight) in get_neighbors(n): open_set.add(m) if m not in open_set and m not in closed_set: parents[m] = n g[m] = g[n] + weight else: if g[m]g[n] + weight: g[m] = g[n] + weight parents[m] = n if m in closed_set: closed_set.remove(m) open_set.add(m) if n == None: print('Path does not exist!") return None if n == stop_node: path = [] while parents[n] != n: path.append(n) n = parents[n] path.append(start_node) path.reverse() print('Path found: {}'.format(path)) return path open_set.remove(n) closed_set.add(n) print('Path does not exist!") return None def get_neighbors(v): if v in Graph_nodes: return Graph_nodes[v] else: return None de...
Read more