docs: def read_images_in_memory(job_model, dataset, node, trainer):
      """
      Reads all images into memory and applies augmentation if enabled
      """
      concurrent = psutil.cpu_count()
  
      dataset_config = dataset['config']
      controller = {'running': True}
      q = Queue(concurrent)
  
      result = {
          'X_train': [],
          'Y_train': [],
          'X_test': [],
          'Y_test': []
      }
  
      images = []
      max = 0
  
      path = job_model.get_dataset_downloads_dir(dataset)
      if 'path' in dataset['config']:
          path = dataset['config']['path']
  
      classes_count = 0
      category_map = {}
      classes = []
  
      trainer.set_status('LOAD IMAGES INTO MEMORY')
  
      try:
          for i in range(concurrent):
              t = ImageReadWorker(q, job_model, node, path, images, controller)
              t.daemon = True
              t.start()
  
          for validation_or_training in ['validation', 'training']:
              if os.path.isdir(os.path.normpath(path + '/' + validation_or_training)):
                  for category_name in os.listdir(os.path.normpath(path + '/' + validation_or_training)):
                      if os.path.isdir(os.path.normpath(path + '/' + validation_or_training + '/' + category_name)):
  
                          if category_name not in category_map:
                              category_map[category_name] = classes_count
                              if 'classes' in dataset_config and 'category_' in category_name:
                                  category_idx = int(category_name.replace('category_', ''))
                                  category_map[category_name] = category_idx
                                  target_category = dataset_config['classes'][category_idx]
                                  classes.append(target_category['title'] or 'Class %s' % (category_idx, ))
                              else:
                                  classes.append(category_name)
  
                              classes_count += 1
  
                          for id in os.listdir(os.path.normpath(path + '/' + validation_or_training + '/' + category_name)):
                              file_path = os.path.join(path, validation_or_training, category_name, id)
                              q.put([file_path, validation_or_training == 'validation', category_name])
                              max += 1
  
          q.join()
          controller['running'] = False
  
          train_images = []
          test_images = []
  
          for v in images:
              image, validation, category_dir = v
              if validation is True:
                  test_images.append([image, category_map[category_dir]])
              else:
                  train_images.append([image, category_map[category_dir]])
  
          train_datagen = None
          augmentation = bool(get_option(dataset_config, 'augmentation', False))
          if augmentation:
              train_datagen = get_image_data_augmentor_from_dataset(dataset)
  
          train = InMemoryDataGenerator(train_datagen, train_images, classes_count, job_model.job['config']['batchSize'])
  
          test = InMemoryDataGenerator(None, test_images, classes_count, job_model.job['config']['batchSize'])
  
          nb_sample = len(train_images)
          trainer.set_info('Dataset size', {'training': nb_sample, 'validation': len(test_images)})
          trainer.set_generator_training_nb(nb_sample)
          trainer.set_generator_validation_nb(len(test_images))
  
          trainer.logger.info(("Found %d classes, %d images (%d in training [%saugmented], %d in validation). Read all images into memory from %s" %
                 (classes_count, max, len(train_images), 'not ' if augmentation is False else '', len(test_images), path)))
  
          if classes_count == 0:
              trainer.logger.warning("Could not find any classes. Does the directory contains images?")
              sys.exit(1)
  
          trainer.output_size = classes_count
          trainer.set_info('classes', classes)
          trainer.classes = classes
  
          result['X_train'] = train
          result['Y_train'] = train
          result['X_test'] = test
          result['Y_test'] = test
  
          return result
  
      except KeyboardInterrupt:
          controller['running'] = False
          sys.exit(1)