Sign In
In [ ]:
import jovian
[jovian] Saving notebook..
In [2]:
!pip install jovian
Collecting jovian Using cached Requirement already satisfied: requests in c:\users\neeraj\anaconda3\envs\\lib\site-packages (from jovian) (2.22.0) Collecting uuid (from jovian) Using cached Requirement already satisfied: chardet<3.1.0,>=3.0.2 in c:\users\neeraj\anaconda3\envs\\lib\site-packages (from requests->jovian) (3.0.4) Requirement already satisfied: certifi>=2017.4.17 in c:\users\neeraj\anaconda3\envs\\lib\site-packages (from requests->jovian) (2019.6.16) Requirement already satisfied: idna<2.9,>=2.5 in c:\users\neeraj\anaconda3\envs\\lib\site-packages (from requests->jovian) (2.8) Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in c:\users\neeraj\anaconda3\envs\\lib\site-packages (from requests->jovian) (1.24.3) Building wheels for collected packages: jovian, uuid Building wheel for jovian ( started Building wheel for jovian ( finished with status 'done' Stored in directory: C:\Users\Neeraj\AppData\Local\pip\Cache\wheels\67\9f\b0\b841f81da3f8c9919ed232fa14bba397c5c38fa8df5d125d8a Building wheel for uuid ( started Building wheel for uuid ( finished with status 'done' Stored in directory: C:\Users\Neeraj\AppData\Local\pip\Cache\wheels\2a\80\9b\015026567c29fdffe31d91edbe7ba1b17728db79194fca1f21 Successfully built jovian uuid Installing collected packages: uuid, jovian Successfully installed jovian-0.1.62 uuid-1.30
In [2]:
%matplotlib inline
In [3]:
import torch
import torch.nn as nn
import torch.nn.functional as F
import torch.optim as optim

<torch._C.Generator at 0x7f53ecc10750>
In [4]:
word_to_ix = {"hello": 0, "world": 1}
embeds = nn.Embedding(2, 5)  # 2 words in vocab, 5 dimensional embeddings
lookup_tensor = torch.tensor([word_to_ix["hello"]], dtype=torch.long)
hello_embed = embeds(lookup_tensor)
tensor([[ 0.6614, 0.2669, 0.0617, 0.6213, -0.4519]], grad_fn=<EmbeddingBackward>)

An Example: N-Gram Language Modeling

[Due to a LaTeX render issue with Jupyter, the supplementary material has been uploaded separately, please check here](

In [5]:
# We will use Shakespeare Sonnet 2
test_sentence = """When forty winters shall besiege thy brow,
And dig deep trenches in thy beauty's field,
Thy youth's proud livery so gazed on now,
Will be a totter'd weed of small worth held:
Then being asked, where all thy beauty lies,
Where all the treasure of thy lusty days;
To say, within thine own deep sunken eyes,
Were an all-eating shame, and thriftless praise.
How much more praise deserv'd thy beauty's use,
If thou couldst answer 'This fair child of mine
Shall sum my count, and make my old excuse,'
Proving his beauty by succession thine!
This were to be new made when thou art old,
And see thy blood warm when thou feel'st it cold.""".split()
# we should tokenize the input, but we will ignore that for now
# build a list of tuples.  Each tuple is ([ word_i-2, word_i-1 ], target word)
trigrams = [([test_sentence[i], test_sentence[i + 1]], test_sentence[i + 2])
            for i in range(len(test_sentence) - 2)]
# print the first 3, just so you can see what they look like

vocab = set(test_sentence)
word_to_ix = {word: i for i, word in enumerate(vocab)}

class NGramLanguageModeler(nn.Module):

    def __init__(self, vocab_size, embedding_dim, context_size):
        super(NGramLanguageModeler, self).__init__()
        self.embeddings = nn.Embedding(vocab_size, embedding_dim)
        self.linear1 = nn.Linear(context_size * embedding_dim, 128)
        self.linear2 = nn.Linear(128, vocab_size)

    def forward(self, inputs):
        embeds = self.embeddings(inputs).view((1, -1))
        out = F.relu(self.linear1(embeds))
        out = self.linear2(out)
        log_probs = F.log_softmax(out, dim=1)
        return log_probs

losses = []
loss_function = nn.NLLLoss()
model = NGramLanguageModeler(len(vocab), EMBEDDING_DIM, CONTEXT_SIZE)
optimizer = optim.SGD(model.parameters(), lr=0.001)

for epoch in range(10):
    total_loss = 0
    for context, target in trigrams:

        # Step 1. Prepare the inputs to be passed to the model (i.e, turn the words
        # into integer indices and wrap them in tensors)
        context_idxs = torch.tensor([word_to_ix[w] for w in context], dtype=torch.long)

        # Step 2. Recall that torch *accumulates* gradients. Before passing in a
        # new instance, you need to zero out the gradients from the old
        # instance

        # Step 3. Run the forward pass, getting log probabilities over next
        # words
        log_probs = model(context_idxs)

        # Step 4. Compute your loss function. (Again, Torch wants the target
        # word wrapped in a tensor)
        loss = loss_function(log_probs, torch.tensor([word_to_ix[target]], dtype=torch.long))

        # Step 5. Do the backward pass and update the gradient

        # Get the Python number from a 1-element Tensor by calling tensor.item()
        total_loss += loss.item()
print(losses)  # The loss decreased every iteration over the training data!
[(['When', 'forty'], 'winters'), (['forty', 'winters'], 'shall'), (['winters', 'shall'], 'besiege')] [521.3776135444641, 518.7374219894409, 516.1145420074463, 513.5080490112305, 510.9162576198578, 508.33981370925903, 505.7793562412262, 503.2336993217468, 500.70192098617554, 498.18147253990173]
In [7]:
def make_context_vector(context, word_to_ix):
    idxs = [word_to_ix[w] for w in context]
    return torch.tensor(idxs, dtype=torch.long)

def get_index_of_max(input):
    index = 0
    for i in range(1, len(input)):
        if input[i] > input[index]:
            index = i 
    return index

def get_max_prob_result(input, ix_to_word):
    return ix_to_word[get_index_of_max(input)]

CONTEXT_SIZE = 2  # 2 words to the left, 2 to the right
In [6]:
CONTEXT_SIZE = 2  # 2 words to the left, 2 to the right
raw_text = """We are about to study the idea of a computational process.
Computational processes are abstract beings that inhabit computers.
As they evolve, processes manipulate other abstract things called data.
The evolution of a process is directed by a pattern of rules
called a program. People create programs to direct processes. In effect,
we conjure the spirits of the computer with our spells.""".split()

# By deriving a set from `raw_text`, we deduplicate the array
vocab = set(raw_text)
vocab_size = len(vocab)

word_to_ix = {word: i for i, word in enumerate(vocab)}
data = []
for i in range(2, len(raw_text) - 2):
    context = [raw_text[i - 2], raw_text[i - 1],
               raw_text[i + 1], raw_text[i + 2]]
    target = raw_text[i]
    data.append((context, target))

class CBOW(nn.Module):

    def __init__(self):

    def forward(self, inputs):

# create your model and train.  here are some functions to help you make
# the data ready for use by your module

def make_context_vector(context, word_to_ix):
    idxs = [word_to_ix[w] for w in context]
    return torch.tensor(idxs, dtype=torch.long)

make_context_vector(data[0][0], word_to_ix)  # example
[(['We', 'are', 'to', 'study'], 'about'), (['are', 'about', 'study', 'the'], 'to'), (['about', 'to', 'the', 'idea'], 'study'), (['to', 'study', 'idea', 'of'], 'the'), (['study', 'the', 'of', 'a'], 'idea')]
tensor([16, 39,  4,  9])