nlp语义理解的一点儿看法_nlp 语义理解差

import collections
import os
import random
import zipfile
import numpy as np
import urllib.request as request
import tensorflow as tf
 
url = 'http://mattmahoney.net/dc/'
 
def maybe_download(filename,expected_bytes):
    if not os.path.exists(filename):
        filename,_ = request.urlretrieve(url+filename,filename)
    statinfo = os.stat(filename)
    if statinfo.st_size == expected_bytes:
        print('Found and verified',filename)
    else:
        print(statinfo.st_size)
        raise Exception('Failed to verify' + filename + '.Can you get to it with a browser?')
    return filename
 
def read_data(filename):
    with zipfile.ZipFile(filename) as f:
        data = tf.compat.as_str(f.read(f.namelist()[0])).split()
    return data
 
vocabulary_size = 50000
def build_dataset(words):
    count = [['UNK',-1]]
    count.extend(collections.Counter(words).most_common(vocabulary_size - 1))
    dictionary = dict(zip(list(zip(*count))[0],range(len(list(zip(*count))[0]))))
    data = list()
    un_count = 0
 
    for word in words:
        if word in dictionary:
            index = dictionary[word]
        else:
            index = 0
            un_count += 1
        data.append(index)
    count[0][1] = un_count
    reverse_dictionary = dict(zip(dictionary.values(),dictionary.keys()))
    return data,reverse_dictionary,dictionary,count
 
data_index = 0
def generate_batch(data,batch_size,num_skips,skip_window):
    filename = maybe_download('text8.zip', 31344016)
    words = read_data(filename)
    global data_index
    assert num_skips <= 2 * skip_window
    assert batch_size % num_skips == 0
    span = 2 * skip_window + 1
    batch = np.ndarray(shape=[batch_size],dtype=np.int32)
    labels = np.ndarray(shape=[batch_size,1],dtype=np.int32)
    buffer = collections.deque(maxlen=span)
    #初始化
    for i in range(span):
        buffer.append(data[data_index])
        data_index = (data_index + 1) % len(data)
    #移动窗口，获取批量数据
    for i in range(batch_size // num_skips):
        target = skip_window
        avoid_target = [skip_window]
        for j in range(num_skips):
            while target in avoid_target:
                target = np.random.randint(0,span)
            avoid_target.append(target)
            batch[i * num_skips + j] = buffer[skip_window]
            labels[i * num_skips + j,0] = buffer[target]
 
        buffer.append(data[data_index])
        data_index = (data_index + 1) % len(data)
    return batch,labels

import tensorflow as tf
import w2v.data_helper as da
import numpy as np
import math
#filename = da.maybe_download('text8.zip', 31344016)
words = da.read_data("text8.zip")
assert  words is not None
data,reverse_dictionary,dictionary,count = da.build_dataset(words)
 
class config(object):
    batch_size = 128
    embedding_size = 128
    skip_window = 1
    num_skips = 2
 
    valid_size = 16
    valid_window = 100
    valid_examples = np.random.choice(valid_window, valid_size, replace=False)
    num_sampled = 64
    vocabulary_size = 50000
    num_steps = 10001
 
class w2vModel(object):
    def __init__(self,config):
        self.train_inputs = train_inputs = tf.placeholder(tf.int32, shape=[config.batch_size])
        self.train_labels = train_labels = tf.placeholder(tf.int32, shape=[config.batch_size, 1])
        self.valid_dataset = valid_dataset = tf.constant(config.valid_examples, dtype=tf.int32)
 
        with tf.device('/cpu:0'):
            embeddings = tf.Variable(
                tf.random_uniform(shape=[config.vocabulary_size, config.embedding_size], minval=-1.0, maxval=1.0))
            embed = tf.nn.embedding_lookup(embeddings, train_inputs)
            nce_weights = tf.Variable(
                tf.truncated_normal([config.vocabulary_size, config.embedding_size], stddev=1.0 / math.sqrt(config.embedding_size)))
            nce_bias = tf.Variable(tf.zeros([config.vocabulary_size]))
 
            loss = tf.reduce_mean(
                tf.nn.nce_loss(weights=nce_weights, biases=nce_bias, labels=train_labels, inputs=embed,
                               num_sampled=config.num_sampled, num_classes=config.vocabulary_size))
            optimizer = tf.train.GradientDescentOptimizer(1.0).minimize(loss)
            norm = tf.sqrt(tf.reduce_sum(tf.square(embeddings), 1, keep_dims=True))
            normalized_embeddings = embeddings / norm
            valid_embeddings = tf.nn.embedding_lookup(normalized_embeddings, valid_dataset)
            similarity = tf.matmul(valid_embeddings, normalized_embeddings, transpose_b=True)
            tf.add_to_collection("embedding",embeddings)
            self.saver = saver = tf.train.Saver(tf.global_variables())

import tensorflow as tf
import w2v.w2vmodel as model
import w2v.data_helper as da
 
config = model.config()
 
with tf.Graph().as_default() as g:
    Model = model.w2vModel(config)
    with tf.Session(graph=g) as session:
        tf.global_variables_initializer().run()
        print("initialized")
 
        average_loss = 0.0
        for step in range(config.num_steps):
            batch_inputs,batch_labels = da.generate_batch(model.data,config.batch_size,config.num_skips,config.skip_window)
            feed_dict = {Model.train_inputs:batch_inputs,Model.train_labels:batch_labels}
 
            _,loss_val = session.run([Model.optimizer,Model.loss],feed_dict=feed_dict)
            average_loss += loss_val
            if step % 2000 == 0:
                if step > 0:
                    average_loss /= 2000
                print("Average loss at step",step,":",average_loss)
                average_loss = 0
            if step % 10000 == 0:
                sim = Model.similarity.eval()
                for i in range(config.valid_size):
                    valid_word = model.reverse_dictionary[config.valid_examples[i]]
                    top_k = 8
                    nearest = (-sim[i,:]).argsort()[1:top_k+1]
                    log_str = "Nearest to %s:" % valid_word
                    for k in range(top_k):
                        close_word = model.reverse_dictionary[nearest[k]]
                        log_str = "%s %s," % (log_str,close_word)
                    print(log_str)
        Model.saver.save(session, "E:/word2vector/models/model.ckpt")
        #final_embeddings = model.normalized_embeddings.eval()