浅谈keras通过model.fit_generator训练模型(节省内存)

fit_generator(generator, 
steps_per_epoch=None, 
epochs=1, 
verbose=1, 
callbacks=None, 
validation_data=None, 
validation_steps=None, 
class_weight=None, 
max_queue_size=10, 
workers=1, 
use_multiprocessing=False, 
shuffle=True, 
initial_epoch=0)

import keras
from keras.models import Sequential
from keras.layers import Dense
import numpy as np
from sklearn.model_selection import train_test_split
from PIL import Image

def process_x(path):
 img = Image.open(path)
 img = img.resize((96,96))
 img = img.convert('RGB')
 img = np.array(img)

 img = np.asarray(img, np.float32) / 255.0
 #也可以进行进行一些数据数据增强的处理
 return img

count =1
def generate_arrays_from_file(x_y):
 #x_y 是我们的训练集包括标签，每一行的第一个是我们的图片路径，后面的是我们的独热化后的标签

 global count
 batch_size = 8
 while 1:
  batch_x = x_y[(count - 1) * batch_size:count * batch_size, 0]
  batch_y = x_y[(count - 1) * batch_size:count * batch_size, 1:]

  batch_x = np.array([process_x(img_path) for img_path in batch_x])
  batch_y = np.array(batch_y).astype(np.float32)
  print("count:"+str(count))
  count = count+1
  yield (batch_x, batch_y)

model = Sequential()
model.add(Dense(units=1000, activation='relu', input_dim=2))
model.add(Dense(units=2, activation='softmax'))
model.compile(loss='categorical_crossentropy',optimizer='sgd',metrics=['accuracy'])

x_y = []
model.fit_generator(generate_arrays_from_file(x_y),steps_per_epoch=10, epochs=2,max_queue_size=1,workers=1)

def foo():
 print("starting...")
 while True:
  res = yield 4
  print("res:",res)
g = foo()
print(next(g))
print("----------")
print(next(g))

starting...
4
----------
res: None
4

class BaseSequence(Sequence):
 """
 基础的数据流生成器，每次迭代返回一个batch
 BaseSequence可直接用于fit_generator的generator参数
 fit_generator会将BaseSequence再次封装为一个多进程的数据流生成器
 而且能保证在多进程下的一个epoch中不会重复取相同的样本
 """
 def __init__(self, img_paths, labels, batch_size, img_size):
  #np.hstack在水平方向上平铺
  self.x_y = np.hstack((np.array(img_paths).reshape(len(img_paths), 1), np.array(labels)))
  self.batch_size = batch_size
  self.img_size = img_size

 def __len__(self):
  #math.ceil表示向上取整
  #调用len(BaseSequence)时返回，返回的是每个epoch我们需要读取数据的次数
  return math.ceil(len(self.x_y) / self.batch_size)

 def preprocess_img(self, img_path):

  img = Image.open(img_path)
  resize_scale = self.img_size[0] / max(img.size[:2])
  img = img.resize((self.img_size[0], self.img_size[0]))
  img = img.convert('RGB')
  img = np.array(img)

  # 数据归一化
  img = np.asarray(img, np.float32) / 255.0
  return img

 def __getitem__(self, idx):
  batch_x = self.x_y[idx * self.batch_size: (idx + 1) * self.batch_size, 0]
  batch_y = self.x_y[idx * self.batch_size: (idx + 1) * self.batch_size, 1:]
  batch_x = np.array([self.preprocess_img(img_path) for img_path in batch_x])
  batch_y = np.array(batch_y).astype(np.float32)
  print(batch_x.shape)
  return batch_x, batch_y
 #重写的父类Sequence中的on_epoch_end方法，在每次迭代完后调用。
 def on_epoch_end(self):
  #每次迭代后重新打乱训练集数据
  np.random.shuffle(self.x_y)

class Animal:
 def __init__(self, animal_list):
  self.animals_name = animal_list

 def __getitem__(self, index):
  return self.animals_name[index]

animals = Animal(["dog","cat","fish"])
for animal in animals:
 print(animal)

dog
cat
fish