Aug-17-2021, 07:54 AM
I have to test word2vector model for text data similarity it generate this kind of error ValueError: dimension mismatch
from keras.preprocessing.text import Tokenizer from keras.preprocessing.sequence import pad_sequences tokenizer=Tokenizer() tokenizer.fit_on_texts(documents_df.documents_cleaned) tokenized_documents=tokenizer.texts_to_sequences(documents_df.documents_cleaned) tokenized_paded_documents=pad_sequences(tokenized_documents,maxlen=64,padding='post') vocab_size=len(tokenizer.word_index)+1 # reading Glove word embeddings into a dictionary with "word" as key and values as word vectors embeddings_index = dict() with open("D:\Clustering\glove.6B.100d.txt", 'r', encoding="utf8") as file: for line in file: values = line.split() word = values[0] coefs = np.asarray(values[1:], dtype='float32') embeddings_index[word] = coefs # creating embedding matrix, every row is a vector representation from the vocabulary indexed by the tokenizer index. embedding_matrix=np.zeros((vocab_size,100)) for word,i in tokenizer.word_index.items(): embedding_vector = embeddings_index.get(word) if embedding_vector is not None: embedding_matrix[i] = embedding_vector # calculating average of word vectors of a document weighted by tf-idf document_embeddings=np.zeros((len(tokenized_paded_documents),100)) words=tfidfvectoriser.get_feature_names() # instead of creating document-word embeddings, directly creating document embeddings for i in range(documents_df.shape[0]): for j in range(len(words)): document_embeddings[i]+=embedding_matrix[tokenizer.word_index[words[j]]]*tfidf_vectors[i][j] pairwise_similarities=cosine_similarity(document_embeddings) pairwise_differences=euclidean_distances(document_embeddings)Error: Error:Traceback (most recent call last): File "D:/Clustering/text-cluster-master/Cos_Sim_Eucliden_distance.py", line 156, in <module> document_embeddings[i] += embedding_matrix[tokenizer.word_index[words[j]]] * tfidf_vectors[i][j] File "D:\Python3.8.0\Python\lib\site-packages\scipy\sparse\base.py", line 550, in __rmul__ return (self.transpose() * tr).transpose() File "D:\Python3.8.0\Python\lib\site-packages\scipy\sparse\base.py", line 498, in __mul__ raise ValueError('dimension mismatch') ValueError: dimension mismatch