9

我正在玩有关 text_analysis 的 Kaggle 比赛的一些数据，每当我尝试适应我的算法时，我都会不断收到标题中描述的这个相当奇怪的错误。我查了一下，这与我的矩阵在呈现为稀疏矩阵时密集地填充了非零元素有关。我认为这个问题出在代码下面的 train_labels 上，标签由 24 列组成，一开始并不常见，标签是 0 和 1 之间的浮点数（包括 0 和 1）。尽管对问题所在有所了解，但我不知道如何正确解决它，而且我之前的尝试也没有那么好。你们对我如何解决这个问题有什么建议吗？

代码：

import numpy as np
import pandas as p
import nltk
from sklearn.feature_extraction.text import TfidfVectorizer
import os
from sklearn.linear_model  import RidgeCV

dir = "C:/Users/Anonymous/Desktop/KAGA FOLDER/Hashtags"

def clean_the_text(data):
    alist = []
    data = nltk.word_tokenize(data)
    for j in data:
        alist.append(j.rstrip('\n'))
    alist = " ".join(alist)

    return alist
def loop_data(data):
    for i in range(len(data)):
        data[i] = clean_the_text(data[i])
    return data      

if __name__ == "__main__":
    print("loading data")
    train_text = loop_data(list(np.array(p.read_csv(os.path.join(dir,"train.csv")))[:,1]))
    test_set = loop_data(list(np.array(p.read_csv(os.path.join(dir,"test.csv")))[:,1]))
    train_labels  = np.array(p.read_csv(os.path.join(dir,"train.csv")))[:,4:]



    #Vectorizing
    vectorizer = TfidfVectorizer(max_features = 10000,strip_accents = "unicode",analyzer = "word")
    ridge_classifier = RidgeCV(alphas = [0.001,0.01,0.1,1,10])
    all_data = train_text + test_set
    train_length  = len(train_text)

    print("fitting Vectorizer")
    vectorizer.fit(all_data)
    print("transforming text")
    all_data = vectorizer.transform(all_data)
    train = all_data[:train_length]
    test = all_data[train_length:]

    print("fitting and selecting models") 
    ridge_classifier.fit(train,train_labels)
    print("predicting")
    pred = ridge_classifier.predict(test)


    np.savetxt(dir +"submission.csv", pred, fmt = "%d", delimiter = ",")
    print("submission_file created")

追溯：

Traceback (most recent call last):
  File "C:\Users\Anonymous\workspace\final_submission\src\linearSVM.py", line 56, in <module>
    ridge_classifier.fit(train,train_labels)
  File "C:\Python27\lib\site-packages\sklearn\linear_model\ridge.py", line 817, in fit
    estimator.fit(X, y, sample_weight=sample_weight)
  File "C:\Python27\lib\site-packages\sklearn\linear_model\ridge.py", line 724, in fit
    v, Q, QT_y = _pre_compute(X, y)
  File "C:\Python27\lib\site-packages\sklearn\linear_model\ridge.py", line 609, in _pre_compute
    K = safe_sparse_dot(X, X.T, dense_output=True)
  File "C:\Python27\lib\site-packages\sklearn\utils\extmath.py", line 78, in safe_sparse_dot
    ret = a * b
  File "C:\Python27\lib\site-packages\scipy\sparse\base.py", line 303, in __mul__
    return self._mul_sparse_matrix(other)
  File "C:\Python27\lib\site-packages\scipy\sparse\compressed.py", line 520, in _mul_sparse_matrix
    indices = np.empty(nnz, dtype=np.intc)
ValueError: negative dimensions are not allowed

我怀疑我的标签是问题，所以这里是标签：

In [12]:
undefined



import pandas as pd
import numpy as np
import os
dir = "C:\Users\Anonymous\Desktop\KAGA FOLDER\Hashtags"
labels = np.array(pd.read_csv(os.path.join(dir,"train.csv")))[:,4:]
labels


Out[12]:
array([[0.0, 0.0, 1.0, ..., 0.0, 0.0, 0.0],
       [0.0, 0.0, 0.0, ..., 0.0, 0.0, 0.0],
       [0.0, 0.0, 0.0, ..., 0.0, 0.0, 0.0],
       ..., 
       [0.0, 0.0, 0.0, ..., 1.0, 0.0, 0.0],
       [0.0, 0.385, 0.41, ..., 0.0, 0.0, 0.0],
       [0.0, 0.20199999999999999, 0.395, ..., 0.0, 0.0, 0.0]], dtype=object)
In [13]:
undefined



labels.shape
Out[13]:
(77946L, 24L)
4

1 回答 1

2

问题是由于尺寸不匹配。

train_labels实际上是所有数据的类。train的大小train_labels应该匹配。

于 2013-11-12T20:14:50.837 回答