[catboost/tutorials] Improve text_embedding_features tutorial

Note: mandatory check (NEED_CHECK) was skipped ref:aca587f2a8b976c46e15f6a132bb19dbae80eca6

Author: nikitxskv

events/2020_11_18_catboost_tutorial/text_embedding_features.ipynb

@@ -526,6 +526,15 @@
  "from catboost.text_processing import Dictionary"
  ]
  },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "text_small_lemmatized_nltk"
+ ]
+ },
  {
  "cell_type": "code",
  "execution_count": null,
@@ -560,6 +569,15 @@
  "!cat dictionary.tsv"
  ]
  },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "dictionary.apply([text_small_lemmatized_nltk[0]])"
+ ]
+ },
  {
  "cell_type": "markdown",
  "metadata": {
@@ -594,35 +612,16 @@
  {
  "cell_type": "code",
  "execution_count": null,
- "metadata": {
- "colab": {
- "base_uri": "https://localhost:8080/",
- "height": 161
- },
- "colab_type": "code",
- "id": "7Ea944JbfFuu",
- "outputId": "5f788c52-345c-4703-957a-4f57dd29c418"
- },
- "outputs": [],
- "source": [
- "text_small_lemmatized_nltk"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {
- "colab": {
- "base_uri": "https://localhost:8080/",
- "height": 35
- },
- "colab_type": "code",
- "id": "bRm5Cf5qkzlJ",
- "outputId": "6226eea1-ab2b-4924-df6c-a006e71965f5"
- },
+ "metadata": {},
  "outputs": [],
  "source": [
- "dictionary.apply([text_small_lemmatized_nltk[0]])"
+ "X_proc_train_small, y_train_small = X_preprocessed_train[:1000]['review'].to_list(), y_train[:1000]\n",
+ "X_proc_train_small = list(map(simple_tokenizer.tokenize, X_proc_train_small))\n",
+ "X_proc_test_small, y_test_small = X_preprocessed_test[:1000]['review'].to_list(), y_test[:1000]\n",
+ "X_proc_test_small = list(map(simple_tokenizer.tokenize, X_proc_test_small))\n",
+ "\n",
+ "dictionary = Dictionary(max_dictionary_size=100)\n",
+ "dictionary.fit(X_proc_train_small);"
  ]
  },
  {
@@ -643,11 +642,13 @@
  " features = np.zeros((len(tokenized_text), dictionary.size))\n",
  " for i, tokenized_sentence in enumerate(tokenized_text):\n",
  " indices = np.array(dictionary.apply([tokenized_sentence])[0])\n",
- " features[i, indices] = 1\n",
+ " if len(indices) > 0:\n",
+ " features[i, indices] = 1\n",
  " return features\n",
  "\n",
- "bow_features = bag_of_words(text_small_lemmatized_nltk, dictionary)\n",
- "bow_features"
+ "X_bow_train_small = bag_of_words(X_proc_train_small, dictionary)\n",
+ "X_bow_test_small = bag_of_words(X_proc_test_small, dictionary)\n",
+ "X_bow_train_small.shape"
  ]
  },
  {
@@ -663,24 +664,17 @@
  "from sklearn.linear_model import LogisticRegression\n",
  "from sklearn.naive_bayes import MultinomialNB\n",
  "from scipy.sparse import csr_matrix\n",
- "from sklearn.metrics import log_loss\n",
+ "from sklearn.metrics import roc_auc_score\n",
  "\n",
- "def fit_linear_model(X, c):\n",
+ "def fit_linear_model(X, y):\n",
  " model = LogisticRegression()\n",
- " model.fit(X, c)\n",
+ " model.fit(X, y)\n",
  " return model\n",
  "\n",
- "def fit_naive_bayes(X, c):\n",
- " clf = MultinomialNB()\n",
- " if isinstance(X, csr_matrix):\n",
- " X.eliminate_zeros()\n",
- " clf.fit(X, c)\n",
- " return clf\n",
- "\n",
- "def evaluate_model_logloss(model, X, y):\n",
+ "def evaluate_model_auc(model, X, y):\n",
  " y_pred = model.predict_proba(X)[:,1]\n",
- " metric = log_loss(y, y_pred)\n",
- " print('Logloss: ' + str(metric))"
+ " metric = roc_auc_score(y, y_pred)\n",
+ " print('AUC: ' + str(metric))"
  ]
  },
  {
@@ -697,19 +691,16 @@
  },
  "outputs": [],
  "source": [
- "def evaluate_models(X, y):\n",
- " linear_model = fit_linear_model(bow_features, target_small)\n",
- " naive_bayes = fit_naive_bayes(bow_features, target_small)\n",
+ "def evaluate_models(X_train, y_train, X_test, y_test):\n",
+ " linear_model = fit_linear_model(X_train, y_train)\n",
  " \n",
  " print('Linear model')\n",
- " evaluate_model_logloss(linear_model, X, y)\n",
- " print('Naive bayes')\n",
- " evaluate_model_logloss(naive_bayes, X, y)\n",
+ " evaluate_model_auc(linear_model, X_test, y_test)\n",
  " print('Comparing to constant prediction')\n",
- " logloss_constant_prediction = log_loss(y, np.ones(shape=(len(text_small), 2)) * 0.5)\n",
- " print('Logloss: ' + str(logloss_constant_prediction))\n",
+ " auc_constant_prediction = roc_auc_score(y_test, np.ones(shape=(len(y_test), 1)) * 0.5)\n",
+ " print('AUC: ' + str(auc_constant_prediction))\n",
  " \n",
- "evaluate_models(bow_features, target_small)"
+ "evaluate_models(X_bow_train_small, y_train_small, X_bow_test_small, y_test_small)"
  ]
  },
  {
@@ -726,11 +717,14 @@
  },
  "outputs": [],
  "source": [
- "dictionary = Dictionary(occurence_lower_bound=0)\n",
- "dictionary.fit(text_small_lemmatized_nltk)\n",
+ "unigram_dictionary = Dictionary(occurence_lower_bound=0, max_dictionary_size=1000)\n",
+ "unigram_dictionary.fit(X_proc_train_small)\n",
  "\n",
- "bow_features = bag_of_words(text_small_lemmatized_nltk, dictionary)\n",
- "evaluate_models(bow_features, target_small)"
+ "X_bow_train_small = bag_of_words(X_proc_train_small, unigram_dictionary)\n",
+ "X_bow_test_small = bag_of_words(X_proc_test_small, unigram_dictionary)\n",
+ "print(X_bow_train_small.shape)\n",
+ "\n",
+ "evaluate_models(X_bow_train_small, y_train_small, X_bow_test_small, y_test_small)"
  ]
  },
  {
@@ -800,8 +794,14 @@
  },
  "outputs": [],
  "source": [
- "bow_features = bag_of_words(text_small_lemmatized_nltk, dictionary)\n",
- "evaluate_models(bow_features, target_small)"
+ "bigram_dictionary = Dictionary(occurence_lower_bound=0, max_dictionary_size=5000, gram_order=2)\n",
+ "bigram_dictionary.fit(X_proc_train_small)\n",
+ "\n",
+ "X_bow_train_small = bag_of_words(X_proc_train_small, bigram_dictionary)\n",
+ "X_bow_test_small = bag_of_words(X_proc_test_small, bigram_dictionary)\n",
+ "print(X_bow_train_small.shape)\n",
+ "\n",
+ "evaluate_models(X_bow_train_small, y_train_small, X_bow_test_small, y_test_small)"
  ]
  },
  {
@@ -828,18 +828,17 @@
  },
  "outputs": [],
  "source": [
- "dictionary1 = Dictionary(occurence_lower_bound=0)\n",
- "dictionary1.fit(text_small_lemmatized_nltk)\n",
+ "X_bow_train_small = np.concatenate((\n",
+ " bag_of_words(X_proc_train_small, unigram_dictionary),\n",
+ " bag_of_words(X_proc_train_small, bigram_dictionary)\n",
+ "), axis=1)\n",
+ "X_bow_test_small = np.concatenate((\n",
+ " bag_of_words(X_proc_test_small, unigram_dictionary),\n",
+ " bag_of_words(X_proc_test_small, bigram_dictionary)\n",
+ "), axis=1)\n",
+ "print(X_bow_train_small.shape)\n",
  "\n",
- "bow_features1 = bag_of_words(text_small_lemmatized_nltk, dictionary1)\n",
- "\n",
- "dictionary2 = Dictionary(occurence_lower_bound=0, gram_order=2)\n",
- "dictionary2.fit(text_small_lemmatized_nltk)\n",
- "\n",
- "bow_features2 = bag_of_words(text_small_lemmatized_nltk, dictionary2)\n",
- "\n",
- "bow_features = np.concatenate((bow_features1, bow_features2), axis=1)\n",
- "evaluate_models(bow_features, target_small)"
+ "evaluate_models(X_bow_train_small, y_train_small, X_bow_test_small, y_test_small)"
  ]
  },
  {
@@ -1066,8 +1065,9 @@
  "metadata": {},
  "outputs": [],
  "source": [
- "X_embed_train_small, y_train_small = X_embed_train[:1000], y_train[:1000]\n",
- "X_embed_test_small, y_test_small = X_embed_test[:1000], y_test[:1000]"
+ "X_embed_first_train_small, y_first_train_small = X_embed_train[:5000], y_train[:5000]\n",
+ "X_embed_second_train_small, y_second_train_small = X_embed_train[5000:10000], y_train[5000:10000]\n",
+ "X_embed_test_small, y_test_small = X_embed_test[:5000], y_test[:5000]"
  ]
  },
  {
@@ -1083,8 +1083,7 @@
  "metadata": {},
  "outputs": [],
  "source": [
- "linmodel = fit_linear_model(X_embed_train_small, y_train_small)\n",
- "evaluate_model_logloss(linmodel, X_embed_test_small, y_test_small)"
+ "evaluate_models(X_embed_second_train_small, y_second_train_small, X_embed_test_small, y_test_small)"
  ]
  },
  {
@@ -1102,18 +1101,13 @@
  "source": [
  "from sklearn.discriminant_analysis import LinearDiscriminantAnalysis\n",
  "\n",
- "clf = LinearDiscriminantAnalysis()\n",
- "clf.fit(X_embed_train_small[:500], y_train_small[:500])\n",
- "\n",
- "X_lda_train_small = clf.transform(X_embed_train_small[500:])\n",
- "X_embed_lda_train_small = np.concatenate([X_embed_train_small[500:], X_lda_train_small], axis=1)\n",
- "\n",
- "X_lda_test_small = clf.transform(X_embed_test_small)\n",
- "X_embed_lda_test_small = np.concatenate([X_embed_test_small, X_lda_test_small], axis=1)\n",
- "\n",
+ "lda = LinearDiscriminantAnalysis(solver='svd')\n",
+ "lda.fit(X_embed_first_train_small, y_first_train_small)\n",
  "\n",
- "linmodel = fit_linear_model(X_embed_lda_train_small, y_train_small[500:])\n",
- "evaluate_model_logloss(linmodel, X_embed_lda_test_small, y_test_small)"
+ "X_lda_train_small = lda.transform(X_embed_second_train_small)\n",
+ "X_lda_test_small = lda.transform(X_embed_test_small)\n",
+ "print(X_lda_train_small.shape)\n",
+ "evaluate_models(X_lda_train_small, y_second_train_small, X_lda_test_small, y_test_small)"
  ]
  },
  {