imdb数据集下载地址: http://ai.stanford.edu/~amaas/data/sentiment/aclImdb_v1.tar.gz

import os
import torch
from torch import nn
import dltools

def read_imdb(data_dir, is_train):
    data, labels = [], []
    for label in ('pos', 'neg'):
        folder_name = os.path.join(data_dir, 'train' if is_train else 'test', label)
        for file in os.listdir(folder_name):   #os.listdir()相当于形成文件夹列表，可以遍历路径下的每一个文件
            with open(os.path.join(folder_name, file), 'rb') as f:
                #文本没有分行，就一次性读取
                review = f.read().decode('utf-8').replace('\n', '')
                data.append(review)
                labels.append(1 if label == 'pos' else 0)  #三目运算符
    return data, labels

data_dir = r'E:\ALOT\10_deep_learning\data\aclImdb'
train_data = read_imdb(data_dir, is_train=True)
print('训练集数目：', len(train_data[0]))

训练集数目： 25000

#列表和元祖没有shape

for x, y in zip(train_data[0][:3], train_data[1][:3]):
    print('标签：', y, 'review:', x[:60])

标签： 1 review: Bromwell High is a cartoon comedy. It ran at the same time a
标签： 1 review: Homelessness (or Houselessness as George Carlin stated) has 
标签： 1 review: Brilliant over-acting by Lesley Ann Warren. Best dramatic ho

#分词
train_tokens = dltools.tokenize(train_data[0], token='word')
train_tokens  #是一个二维列表，  一个文本文件一个（内置）列表

#构建词汇表vocab
vocab = dltools.Vocab(train_tokens, min_freq=5, reserved_tokens=['<pad>'])

len(vocab)

49347

#绘制直方图查看每条评论的字数分布
dltools.set_figsize()
dltools.plt.xlabel('# tokens per review')
dltools.plt.ylabel('count')
dltools.plt.hist([len(line) for line in train_tokens], bins=range(0,1000, 50))  #分成50份