Lab 6
Business Scenario
Welcome!
Today, your manager has assigned you a task to build and compare LSTM and GRU models with the Simple RNN from previous lab using the same preprocessed dataset. Your goal is to check whether these models can better understand long customer queries, improve prediction accuracy, and reduce incorrect query routing before moving toward production.
Git Pull
git pull origin branchNamePre-Lab Preparation
Topic : Long Short-term Memory
1) LSTM gating mechanism
2) GRU gating mechanism
3) Memory Retention
Long Short-term Memory
LSTM is a specialized type of RNN designed to learn long-term dependencies in sequential data. It uses a memory cell and a set of gates to control what information should be retained, updated, or discarded, helping overcome the vanishing gradient problem of basic RNNs.
Gated Recurrent Unit
GRU is a type of gated RNN designed to capture long-term dependencies while using a simpler architecture than LSTM. It uses two gates to control information flow and combines the hidden state and memory functionality into a single state, reducing computational complexity.
Task 1: Reload Lab 4's Preprocessed Dataset & RNN Baseline
import re
import numpy as np, pandas as pd
import nltk
from nltk.corpus import stopwords
from nltk.stem import WordNetLemmatizer
from nltk.tokenize import word_tokenize
import tensorflow as tf
from tensorflow import keras
from keras.layers import TextVectorization, Embedding, LSTM, GRU, Dense
from keras import Sequential
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import LabelEncoder
RANDOM_SEED = 42
DATA_PATH = "smartcart_customer_queries.csv"
MAX_TOKENS = 2000
EMBEDDING_DIM = 16
RNN_UNITS = 16
TEST_SIZE = 0.2
EPOCHS = 30
BATCH_SIZE = 32
Dataset :
Dataset :
import re
import numpy as np, pandas as pd
import nltk
from nltk.corpus import stopwords
from nltk.stem import WordNetLemmatizer
from nltk.tokenize import word_tokenize
import tensorflow as tf
from tensorflow import keras
from keras.layers import TextVectorization, Embedding, LSTM, GRU, Dense
from keras import Sequential
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import LabelEncoder
RANDOM_SEED = 42
DATA_PATH = "smartcart_customer_queries.csv"
MAX_TOKENS = 2000
EMBEDDING_DIM = 16
RNN_UNITS = 16
TEST_SIZE = 0.2
EPOCHS = 30
BATCH_SIZE = 32
np.random.seed(RANDOM_SEED)
tf.random.set_seed(RANDOM_SEED)
nltk.download("punkt"); nltk.download("punkt_tab")
nltk.download("stopwords"); nltk.download("wordnet"); nltk.download("omw-1.4")
lemmatizer = WordNetLemmatizer()
stop_words = set(stopwords.words("english"))
def preprocess(text: str) -> str:
"""Clean, tokenize, drop stopwords, and lemmatize one query."""
text = str(text).lower()
text = re.sub(r'[^a-z0-9\s]', ' ', text)
text = re.sub(r'\s+', ' ', text).strip()
tokens = word_tokenize(text)
tokens = [t for t in tokens if t not in stop_words]
tokens = [lemmatizer.lemmatize(t) for t in tokens]
return ' '.join(tokens)
df = pd.read_csv(DATA_PATH)
df['processed_text'] = df['query_text'].apply(preprocess)
train_df, test_df = train_test_split(
df, test_size=TEST_SIZE, random_state=RANDOM_SEED, stratify=df['category'])
sequence_length = int(np.percentile(
train_df['processed_text'].str.split().apply(len), 95))
int_vectorizer = TextVectorization(
output_mode='int', max_tokens=MAX_TOKENS, output_sequence_length=sequence_length)
int_vectorizer.adapt(train_df['processed_text'].tolist())
import re
import numpy as np, pandas as pd
import nltk
from nltk.corpus import stopwords
from nltk.stem import WordNetLemmatizer
from nltk.tokenize import word_tokenize
import tensorflow as tf
from tensorflow import keras
from keras.layers import TextVectorization, Embedding, LSTM, GRU, Dense
from keras import Sequential
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import LabelEncoder
RANDOM_SEED = 42
DATA_PATH = "smartcart_customer_queries.csv"
MAX_TOKENS = 2000
EMBEDDING_DIM = 16
RNN_UNITS = 16
TEST_SIZE = 0.2
EPOCHS = 30
BATCH_SIZE = 32
np.random.seed(RANDOM_SEED)
tf.random.set_seed(RANDOM_SEED)
nltk.download("punkt"); nltk.download("punkt_tab")
nltk.download("stopwords"); nltk.download("wordnet"); nltk.download("omw-1.4")
lemmatizer = WordNetLemmatizer()
stop_words = set(stopwords.words("english"))
def preprocess(text: str) -> str:
"""Clean, tokenize, drop stopwords, and lemmatize one query."""
text = str(text).lower()
text = re.sub(r'[^a-z0-9\s]', ' ', text)
text = re.sub(r'\s+', ' ', text).strip()
tokens = word_tokenize(text)
tokens = [t for t in tokens if t not in stop_words]
tokens = [lemmatizer.lemmatize(t) for t in tokens]
return ' '.join(tokens)
df = pd.read_csv(DATA_PATH)
df['processed_text'] = df['query_text'].apply(preprocess)
train_df, test_df = train_test_split(
df, test_size=TEST_SIZE, random_state=RANDOM_SEED, stratify=df['category'])
sequence_length = int(np.percentile(
train_df['processed_text'].str.split().apply(len), 95))
int_vectorizer = TextVectorization(
output_mode='int', max_tokens=MAX_TOKENS, output_sequence_length=sequence_length)
int_vectorizer.adapt(train_df['processed_text'].tolist())
label_encoder = LabelEncoder().fit(df['category'])
class_names = label_encoder.classes_
X_train = int_vectorizer(train_df['processed_text'].tolist()).numpy()
y_train = label_encoder.transform(train_df['category'])
X_test = int_vectorizer(test_df['processed_text'].tolist()).numpy()
y_test = label_encoder.transform(test_df['category'])
vocab_size = int_vectorizer.vocabulary_size()
# Lab 4 RNN baseline, for direct comparison in Task 5
rnn_baseline = {"short_query_accuracy": 0.881, "long_query_accuracy": 0.67}
print(f"Train/test split: {len(train_df)} / {len(test_df)} queries")
print(f"Sequence length (training cutoff): {sequence_length}")
print(f"Vocabulary size: {vocab_size}")
Output
Task 2 : Build a LSTM Model
def build_lstm_model(vocab_size: int, num_classes: int) -> keras.Model:
model = Sequential([
Embedding(input_dim=vocab_size, output_dim=EMBEDDING_DIM),
LSTM(RNN_UNITS),
Dense(num_classes, activation='softmax')])
model.compile(
optimizer='adam',
loss='sparse_categorical_crossentropy',
metrics=['accuracy'])
return model
lstm_model = build_lstm_model(vocab_size, num_classes=len(class_names))
lstm_model.summary()Output
Task 3 : Build a GRU Model
def build_gru_model(vocab_size: int, num_classes: int) -> keras.Model:
model = Sequential([
Embedding(input_dim=vocab_size, output_dim=EMBEDDING_DIM),
GRU(RNN_UNITS),
Dense(num_classes, activation='softmax')])
model.compile(
optimizer='adam',
loss='sparse_categorical_crossentropy',
metrics=['accuracy'])
return model
gru_model = build_gru_model(vocab_size, num_classes=len(class_names))
gru_model.summary()Output
Task 4: Train and Evaluate Both Models on Short Queries
early_stop = keras.callbacks.EarlyStopping(
monitor='val_accuracy', patience=5, restore_best_weights=True
)
results = {}
for name, model in [("LSTM", lstm_model), ("GRU", gru_model)]:
history = model.fit(
X_train, y_train, validation_split=0.2,
epochs=EPOCHS, batch_size=BATCH_SIZE,
callbacks=[early_stop], verbose=0
)
loss, acc = model.evaluate(X_test, y_test, verbose=0)
results[name] = acc
print(f"{name}: stopped after {len(history.history['loss'])} epochs, "
f"held-out test accuracy: {acc:.3f}")
print(f"\nSimple RNN (Lab 4 baseline): {rnn_baseline['short_query_accuracy']:.3f}")Output
Task 5: Test Both Models on Longer Queries
long_query_samples = [
{"ticket_id": "SC-4831-01", "expected_category": "Electronics",
"query": "looking for a lightweight wireless bluetooth headphone "
"with good noise cancelling and fast charging under budget "
"two thousand rupee price and good quality"},
{"ticket_id": "SC-4831-02", "expected_category": "Electronics",
"query": "need a smartphone with good camera fast charging under "
"budget five thousand rupee price and best quality and "
"good resolution"},
{"ticket_id": "SC-4831-03", "expected_category": "Apparel",
"query": "want a slim fit cotton casual shirt in navy blue and grey "
"color with good size for men and good quality and stylish"},
{"ticket_id": "SC-4831-04", "expected_category": "Apparel",
"query": "looking for running shoe for woman with good fit and "
"lightweight and stylish and good quality for casual wear"},
{"ticket_id": "SC-4831-05", "expected_category": "Grocery",
"query": "want organic basmati rice pack with good quality and "
"fresh and natural and no cheap packaging and good price"},
{"ticket_id": "SC-4831-06", "expected_category": "Grocery",
"query": "looking for cold pressed olive oil bottle with good "
"quality and fresh and natural and premium and good price"},
]long_query_samples = [
{"ticket_id": "SC-4831-01", "expected_category": "Electronics",
"query": "looking for a lightweight wireless bluetooth headphone "
"with good noise cancelling and fast charging under budget "
"two thousand rupee price and good quality"},
{"ticket_id": "SC-4831-02", "expected_category": "Electronics",
"query": "need a smartphone with good camera fast charging under "
"budget five thousand rupee price and best quality and "
"good resolution"},
{"ticket_id": "SC-4831-03", "expected_category": "Apparel",
"query": "want a slim fit cotton casual shirt in navy blue and grey "
"color with good size for men and good quality and stylish"},
{"ticket_id": "SC-4831-04", "expected_category": "Apparel",
"query": "looking for running shoe for woman with good fit and "
"lightweight and stylish and good quality for casual wear"},
{"ticket_id": "SC-4831-05", "expected_category": "Grocery",
"query": "want organic basmati rice pack with good quality and "
"fresh and natural and no cheap packaging and good price"},
{"ticket_id": "SC-4831-06", "expected_category": "Grocery",
"query": "looking for cold pressed olive oil bottle with good "
"quality and fresh and natural and premium and good price"},
]
train_vocab = set(w for t in train_df['processed_text'] for w in t.split())
for s in long_query_samples:
proc = preprocess(s['query']).split()
known = [w for w in proc if w in train_vocab]
print(f"{s['ticket_id']}: {len(known)}/{len(proc)} words in vocab "
f"({len(known)/len(proc):.0%})")
eval_vectorizer = TextVectorization(output_mode='int', max_tokens=MAX_TOKENS)
eval_vectorizer.adapt(train_df['processed_text'].tolist())
def evaluate_on_queries(model, vectorizer, samples):
texts = [preprocess(s['query']) for s in samples]
sequences = vectorizer(texts).numpy()
probs = model.predict(sequences, verbose=0)
out = pd.DataFrame(samples)
out['predicted_category'] = class_names[np.argmax(probs, axis=1)]
out['confidence'] = probs.max(axis=1).round(2)
return out
print()
for name, model in [("LSTM", lstm_model), ("GRU", gru_model)]:
out = evaluate_on_queries(model, eval_vectorizer, long_query_samples)
acc = (out['predicted_category'] == out['expected_category']).mean()
print(f"--- {name} ---")
print(out[['ticket_id', 'expected_category', 'predicted_category', 'confidence']])
print(f"{name} accuracy on SC-4831 long queries: {acc:.2f}\n")
Output
Git Push
git push origin branchNameNext-Lab Preparation
Topic : Transformers : BERT vs GPT
1) Limitations of LSTM / RNN
2) Transformer Architecture
3) Architectural difference between BERT & GPT
Great job!
You have successfully built and evaluated LSTM and GRU classifiers and configuration, confirmed improved performance over the Simple RNN on short and long queries and explored how gating preserves early information and addresses vanishing gradients and limited long-range context.
Checkpoint