Pytorch - Google colaboratory - PIL - 加载批次时操作系统读取错误

Question

我已经安装了 torchvision 所需的 PIL 4.1.1。如何解决这个 Google colaboratory PIL OS 错误？我已将笔记本附带的版本 4.0.0 替换为 >=4.1.1，这是 torchvision 所需的。我还重新启动了运行时。但是，这个问题仍然存在。我已经尝试了多个版本的 PIL，直到最新版本 5.3.0。

def train_model(model, criterion, optimizer, scheduler, num_epochs=25):
    since = time.time()

    best_model_wts = copy.deepcopy(model.state_dict())
    best_acc = 0.0

    for epoch in range(num_epochs):
        print('Epoch {}/{}'.format(epoch, num_epochs - 1))
        print('-' * 10)

        # Each epoch has a training and validation phase
        for phase in ['train', 'valid']:
            if phase == 'train':
                scheduler.step()
                model.train()  # Set model to training mode
            else:
                model.eval()   # Set model to evaluate mode

            running_loss = 0.0
            running_corrects = 0

            # Iterate over data.
            for inputs, labels in dataloaders[phase]:
                inputs = inputs.to(device)
                labels = labels.to(device)

                # zero the parameter gradients
                optimizer.zero_grad()

                # forward
                # track history if only in train
                with torch.set_grad_enabled(phase == 'train'):
                    outputs = model(inputs)
                    _, preds = torch.max(outputs, 1)
                    loss = criterion(outputs, labels)

                    # backward + optimize only if in training phase
                    if phase == 'train':
                        loss.backward()
                        optimizer.step()

                # statistics
                running_loss += loss.item() * inputs.size(0)
                running_corrects += torch.sum(preds == labels.data)

            epoch_loss = running_loss / dataset_sizes[phase]
            epoch_acc = running_corrects.double() / dataset_sizes[phase]

            print('{} Loss: {:.4f} Acc: {:.4f}'.format(
                phase, epoch_loss, epoch_acc))

            # deep copy the model
            if phase == 'valid' and epoch_acc > best_acc:
                print('New best accuracy')
                best_acc = epoch_acc
                best_model_wts = copy.deepcopy(model.state_dict())
                torch.save(best_model_wts, 'best_model_latest.pth')
            e.write(str(epoch))

        print()

    time_elapsed = time.time() - since
    print('Training complete in {:.0f}m {:.0f}s'.format(
        time_elapsed // 60, time_elapsed % 60))
    print('Best val Acc: {:4f}'.format(best_acc))

    # load best model weights
    model.load_state_dict(best_model_wts)
    return model
model = train_model(model, criterion,optimizer, exp_lr_scheduler, num_epochs=25)
e.close()

OSError                                   Traceback (most recent call last)
<ipython-input-12-18fb6bbc6b2f> in <module>()
     68     model.load_state_dict(best_model_wts)
     69     return model
---> 70 model = train_model(model, criterion,optimizer, exp_lr_scheduler, num_epochs=25)
     71 e.close()

<ipython-input-12-18fb6bbc6b2f> in train_model(model, criterion, optimizer, scheduler, num_epochs)
     21 
     22             # Iterate over data.
---> 23             for inputs, labels in dataloaders[phase]:
     24                 inputs = inputs.to(device)
     25                 labels = labels.to(device)

/usr/local/lib/python3.6/dist-packages/torch/utils/data/dataloader.py in __next__(self)
    284                 self.reorder_dict[idx] = batch
    285                 continue
--> 286             return self._process_next_batch(batch)
    287 
    288     next = __next__  # Python 2 compatibility

/usr/local/lib/python3.6/dist-packages/torch/utils/data/dataloader.py in _process_next_batch(self, batch)
    305         self._put_indices()
    306         if isinstance(batch, ExceptionWrapper):
--> 307             raise batch.exc_type(batch.exc_msg)
    308         return batch
    309 

OSError: Traceback (most recent call last):
  File "/usr/local/lib/python3.6/dist-packages/torch/utils/data/dataloader.py", line 57, in _worker_loop
    samples = collate_fn([dataset[i] for i in batch_indices])
  File "/usr/local/lib/python3.6/dist-packages/torch/utils/data/dataloader.py", line 57, in <listcomp>
    samples = collate_fn([dataset[i] for i in batch_indices])
  File "/usr/local/lib/python3.6/dist-packages/torchvision/datasets/folder.py", line 101, in __getitem__
    sample = self.loader(path)
  File "/usr/local/lib/python3.6/dist-packages/torchvision/datasets/folder.py", line 147, in default_loader
    return pil_loader(path)
  File "/usr/local/lib/python3.6/dist-packages/torchvision/datasets/folder.py", line 129, in pil_loader
    img = Image.open(f)
  File "/usr/local/lib/python3.6/dist-packages/PIL/Image.py", line 2419, in open
    prefix = fp.read(16)
OSError: [Errno 5] Input/output error

Answer 1

我认为问题与 PIL 的版本无关，而是源于我的代码中的这一行，只要 PIL.version >= 4.1.1

dataloaders = {x:torch.utils.data.DataLoader(image_datasets[x], batch_size=batch_size, num_workers=10) for x in ('train', "valid")}

删除 num_workers 可选参数解决了该问题。也就是说，我将数据加载器声明如下

dataloaders = {x:torch.utils.data.DataLoader(image_datasets[x], batch_size=batch_size) for x in ('train', "valid")}