2

我试图用深度学习来预测异常。我将异常样本标记为 1，将非异常样本标记为 0。在这里，我尝试构建一个模型来获得此链接中定义的一类自动编码器。首先，我为非异常数据训练了自动编码器，并选择最大重建误差作为阈值。然后，我根据这个阈值评估了测试数据。但不幸的是，所有测试样本都被标记为非异常。

from sklearn.metrics import precision_recall_fscore_support as score
import pandas as pd
from sklearn import preprocessing
from sklearn.model_selection import train_test_split
import numpy as np, tensorflow as tf
from sklearn import metrics

TEST_SIZE = 0.2

# DEEP_LEARNING_PARAMETERS
LEARNING_RATE = 0.01
STEP_NUMBER = 5000
UNITS_OF_HIDDEN_LAYER_1 = 256
UNITS_OF_HIDDEN_LAYER_2 = 64
DISPLAY_STEP = int(STEP_NUMBER/10)
KEEPING_PROBABILITY = 0.8
BATCH_PROBABILITY = 0.01

ANOMALY_LABEL = 1.0
NON_ANOMALY_LABEL = 0.0

def build_graph(input_number):

    # Placeholders
    input = tf.placeholder(tf.float32, shape=[None, input_number])

    # Layer 1
    w1 = tf.Variable(tf.truncated_normal([input_number, UNITS_OF_HIDDEN_LAYER_1], stddev=0.05))
    z1 = tf.matmul(input, w1)
    l1 = tf.nn.tanh(z1)
    l1_dropout = tf.nn.dropout(l1, KEEPING_PROBABILITY)

    # Layer 2
    w2 = tf.Variable(tf.truncated_normal([UNITS_OF_HIDDEN_LAYER_1, UNITS_OF_HIDDEN_LAYER_2], stddev=0.05))
    z2 = tf.matmul(l1_dropout, w2)
    l2 = tf.nn.tanh(z2)
    l2_dropout = tf.nn.dropout(l2, KEEPING_PROBABILITY)

    # Layer 3
    w3 = tf.Variable(tf.truncated_normal([UNITS_OF_HIDDEN_LAYER_2, input_number], stddev=0.05))
    b3 = tf.Variable(tf.zeros([input_number]))
    l3 = tf.matmul(l2_dropout, w3) + b3
    y_pred = tf.nn.tanh(l3)

    loss = tf.square(y_pred - input)
    cost = tf.reduce_mean(loss)
    train_step = tf.train.AdamOptimizer(LEARNING_RATE).minimize(cost)

    return input, train_step, y_pred, loss, cost, tf.train.Saver()

def run_model(x_train, y_train, x_test, y_test, input_number):
    acc = []
    train_target = np.asarray(y_train).reshape(-1,1)
    test_target = np.asarray(y_test).reshape(-1,1)

    tf.reset_default_graph()
    input, train_step, y_pred, loss, cost, saver = build_graph(input_number)

    price_list = []
    test_predicted = []
    threshold_loss = -1000000
    TRAIN_NUMBER = x_train.shape[0]
    if(TRAIN_NUMBER == 0):
        print("TOTAL TRAIN IS ZERO, MODEL CANNOT BE GENERATED")
    else:
        with tf.Session() as sess:
            sess.run(tf.global_variables_initializer())
            for i in range(STEP_NUMBER):
                sample = np.random.randint(TRAIN_NUMBER, size=int(10))
                batch_xs = x_train[sample][:]
                batch_ys = train_target[sample][:]
                train_step.run(session=sess, feed_dict={input: batch_xs})
                if i % DISPLAY_STEP == 0:
                    print("i: ", str(i) + " cost: " + str(sess.run([cost], feed_dict={input: x_train})[0]))
                if i == STEP_NUMBER - 1:
                    total_train = np.asarray(x_train).shape[0]
                    for k in range(0, total_train):
                        train_loss = sess.run([cost], feed_dict={input: np.asarray(x_train[k][:]).reshape(1,input_number)})[0]
                        if(train_loss>threshold_loss):
                            threshold_loss = train_loss
                    print("threshold_loss: ", str(np.round(train_loss, 8)))
                    total_test = np.asarray(x_test).shape[0]
                    for k in range(0, total_test):
                        test_loss = sess.run([cost], feed_dict={input: np.asarray(x_test[k][:]).reshape(1,input_number)})[0]
                        temp.append(100*(1-(test_loss/threshold_loss)))
                        if(test_loss>threshold_loss):
                            test_predicted.append(1.0)
                        else:
                            test_predicted.append(0.0)

    accuracy = np.round(sum(np.asarray(test_predicted).reshape(-1,1)==np.asarray(test_target).reshape(-1,1))/len(test_predicted), 4)
    conf_matrix = metrics.confusion_matrix(np.asarray(test_target).reshape(-1,1), np.asarray(test_predicted).reshape(-1,1))
    tn, fp, fn, tp = conf_matrix.ravel()

    precision, recall, fscore, _ = score(np.asarray(test_target).reshape(-1,1), np.asarray(test_predicted).reshape(-1,1), average='binary')
    print("TEST ACCURACY: ", accuracy)
    print("PRECISION: ", precision)
    print("RECALL: ", recall)
    print("FSCORE: ", fscore)

    print("TN: ", tn)
    print("TP: ", tp)
    print("FP: ", fp)
    print("FN: ", fn)

    return accuracy

data = pd.read_csv("test.txt", sep=",")
row, column = data.shape

label = data.values[:,-1]
input = data.values[:,0:column-1]

x_train, x_test, y_train, y_test = train_test_split(input, label, train_size=0.7, shuffle=False)

filtered_input = x_train[np.where(y_train==0.0)]
y_train = y_train[np.where(y_train==0.0)]

scaler = preprocessing.Normalizer()
x_train = scaler.transform(filtered_input)
x_test = scaler.transform(x_test)

run_model(x_train, y_train, x_test, y_test)
4

0 回答 0