/gpfs/projects/LynchGroup/spacewhale/git_spacewhale/spacewhale/shell_scripts
Mon May 20 20:49:34 EDT 2019
Mon May 20 20:49:34 EDT 2019
Now testing Resnext lr=0.1
/gpfs/projects/LynchGroup/spacewhale/git_spacewhale/spacewhale
<torch.utils.data.dataloader.DataLoader object at 0x2aaafe008358>
epoch_23.pth
{0: 'water', 1: 'whale'}
1
2
Exception ignored in: <function _DataLoaderIter.__del__ at 0x2aaaf4309378>
Traceback (most recent call last):
  File "/gpfs/projects/LynchGroup/spacewhale/space_env/lib/python3.7/site-packages/torch/utils/data/dataloader.py", line 399, in __del__
    self._shutdown_workers()
  File "/gpfs/projects/LynchGroup/spacewhale/space_env/lib/python3.7/site-packages/torch/utils/data/dataloader.py", line 378, in _shutdown_workers
    self.worker_result_queue.get()
  File "/gpfs/projects/LynchGroup/spacewhale/space_env/lib/python3.7/multiprocessing/queues.py", line 354, in get
    return _ForkingPickler.loads(res)
  File "/gpfs/projects/LynchGroup/spacewhale/space_env/lib/python3.7/site-packages/torch/multiprocessing/reductions.py", line 151, in rebuild_storage_fd
    fd = df.detach()
  File "/gpfs/projects/LynchGroup/spacewhale/space_env/lib/python3.7/multiprocessing/resource_sharer.py", line 57, in detach
    with _resource_sharer.get_connection(self._id) as conn:
  File "/gpfs/projects/LynchGroup/spacewhale/space_env/lib/python3.7/multiprocessing/resource_sharer.py", line 87, in get_connection
    c = Client(address, authkey=process.current_process().authkey)
  File "/gpfs/projects/LynchGroup/spacewhale/space_env/lib/python3.7/multiprocessing/connection.py", line 499, in Client
    deliver_challenge(c, authkey)
  File "/gpfs/projects/LynchGroup/spacewhale/space_env/lib/python3.7/multiprocessing/connection.py", line 729, in deliver_challenge
    response = connection.recv_bytes(256)        # reject large message
  File "/gpfs/projects/LynchGroup/spacewhale/space_env/lib/python3.7/multiprocessing/connection.py", line 216, in recv_bytes
    buf = self._recv_bytes(maxlength)
  File "/gpfs/projects/LynchGroup/spacewhale/space_env/lib/python3.7/multiprocessing/connection.py", line 407, in _recv_bytes
    buf = self._recv(4)
  File "/gpfs/projects/LynchGroup/spacewhale/space_env/lib/python3.7/multiprocessing/connection.py", line 379, in _recv
    chunk = read(handle, remaining)
ConnectionResetError: [Errno 104] Connection reset by peer
tensor([[[[-1.9467, -1.9467, -1.9467,  ..., -1.9295, -1.9295, -1.9295],
          [-1.9467, -1.9467, -1.9467,  ..., -1.9295, -1.9295, -1.9295],
          [-1.9467, -1.9467, -1.9467,  ..., -1.9295, -1.9295, -1.9295],
          ...,
          [-1.9809, -1.9809, -1.9809,  ..., -1.9124, -1.9124, -1.9124],
          [-1.9809, -1.9809, -1.9809,  ..., -1.9124, -1.9124, -1.9124],
          [-1.9809, -1.9809, -1.9809,  ..., -1.9124, -1.9124, -1.9124]],

         [[-1.5980, -1.5980, -1.5980,  ..., -1.5805, -1.5805, -1.5805],
          [-1.5980, -1.5980, -1.5980,  ..., -1.5805, -1.5805, -1.5805],
          [-1.5980, -1.5980, -1.5980,  ..., -1.5805, -1.5805, -1.5805],
          ...,
          [-1.6155, -1.6155, -1.6155,  ..., -1.5805, -1.5805, -1.5805],
          [-1.6155, -1.6155, -1.6155,  ..., -1.5805, -1.5805, -1.5805],
          [-1.6155, -1.6155, -1.6155,  ..., -1.5805, -1.5805, -1.5805]],

         [[-1.3687, -1.3687, -1.3687,  ..., -1.3687, -1.3687, -1.3687],
          [-1.3687, -1.3687, -1.3687,  ..., -1.3687, -1.3687, -1.3687],
          [-1.3687, -1.3687, -1.3687,  ..., -1.3687, -1.3687, -1.3687],
          ...,
          [-1.3861, -1.3861, -1.3861,  ..., -1.3687, -1.3687, -1.3687],
          [-1.3861, -1.3861, -1.3861,  ..., -1.3687, -1.3687, -1.3687],
          [-1.3861, -1.3861, -1.3861,  ..., -1.3687, -1.3687, -1.3687]]],


        [[[-1.9467, -1.9467, -1.9467,  ..., -1.9467, -1.9467, -1.9467],
          [-1.9467, -1.9467, -1.9467,  ..., -1.9467, -1.9467, -1.9467],
          [-1.9467, -1.9467, -1.9467,  ..., -1.9467, -1.9467, -1.9467],
          ...,
          [-1.9467, -1.9467, -1.9467,  ..., -1.9638, -1.9638, -1.9638],
          [-1.9467, -1.9467, -1.9467,  ..., -1.9638, -1.9638, -1.9638],
          [-1.9467, -1.9467, -1.9467,  ..., -1.9638, -1.9638, -1.9638]],

         [[-1.5980, -1.5980, -1.5980,  ..., -1.5980, -1.5980, -1.5980],
          [-1.5980, -1.5980, -1.5980,  ..., -1.5980, -1.5980, -1.5980],
          [-1.5980, -1.5980, -1.5980,  ..., -1.5980, -1.5980, -1.5980],
          ...,
          [-1.5980, -1.5980, -1.5980,  ..., -1.6155, -1.6155, -1.6155],
          [-1.5980, -1.5980, -1.5980,  ..., -1.6155, -1.6155, -1.6155],
          [-1.5980, -1.5980, -1.5980,  ..., -1.6155, -1.6155, -1.6155]],

         [[-1.3861, -1.3861, -1.3861,  ..., -1.3687, -1.3687, -1.3687],
          [-1.3861, -1.3861, -1.3861,  ..., -1.3687, -1.3687, -1.3687],
          [-1.3861, -1.3861, -1.3861,  ..., -1.3687, -1.3687, -1.3687],
          ...,
          [-1.3687, -1.3687, -1.3687,  ..., -1.3861, -1.3861, -1.3861],
          [-1.3687, -1.3687, -1.3687,  ..., -1.3861, -1.3861, -1.3861],
          [-1.3687, -1.3687, -1.3687,  ..., -1.3861, -1.3861, -1.3861]]],


        [[[-1.9295, -1.9295, -1.9295,  ..., -1.9467, -1.9467, -1.9467],
          [-1.9295, -1.9295, -1.9295,  ..., -1.9467, -1.9467, -1.9467],
          [-1.9295, -1.9295, -1.9295,  ..., -1.9467, -1.9467, -1.9467],
          ...,
          [-1.9295, -1.9295, -1.9295,  ..., -1.9467, -1.9467, -1.9467],
          [-1.9295, -1.9295, -1.9295,  ..., -1.9467, -1.9467, -1.9467],
          [-1.9295, -1.9295, -1.9295,  ..., -1.9467, -1.9467, -1.9467]],

         [[-1.5805, -1.5805, -1.5805,  ..., -1.5980, -1.5980, -1.5980],
          [-1.5805, -1.5805, -1.5805,  ..., -1.5980, -1.5980, -1.5980],
          [-1.5805, -1.5805, -1.5805,  ..., -1.5980, -1.5980, -1.5980],
          ...,
          [-1.5980, -1.5980, -1.5980,  ..., -1.5805, -1.5805, -1.5805],
          [-1.5980, -1.5980, -1.5980,  ..., -1.5805, -1.5805, -1.5805],
          [-1.5980, -1.5980, -1.5980,  ..., -1.5805, -1.5805, -1.5805]],

         [[-1.3513, -1.3513, -1.3513,  ..., -1.3861, -1.3861, -1.3861],
          [-1.3513, -1.3513, -1.3513,  ..., -1.3861, -1.3861, -1.3861],
          [-1.3513, -1.3513, -1.3513,  ..., -1.3861, -1.3861, -1.3861],
          ...,
          [-1.3687, -1.3687, -1.3687,  ..., -1.3687, -1.3687, -1.3687],
          [-1.3687, -1.3687, -1.3687,  ..., -1.3687, -1.3687, -1.3687],
          [-1.3687, -1.3687, -1.3687,  ..., -1.3687, -1.3687, -1.3687]]],


        ...,


        [[[-1.9467, -1.9467, -1.9467,  ..., -1.9467, -1.9467, -1.9467],
          [-1.9467, -1.9467, -1.9467,  ..., -1.9467, -1.9467, -1.9467],
          [-1.9467, -1.9467, -1.9467,  ..., -1.9467, -1.9467, -1.9467],
          ...,
          [-1.9295, -1.9295, -1.9295,  ..., -1.9124, -1.9124, -1.9124],
          [-1.9295, -1.9295, -1.9295,  ..., -1.9124, -1.9124, -1.9124],
          [-1.9295, -1.9295, -1.9295,  ..., -1.9124, -1.9124, -1.9124]],

         [[-1.5630, -1.5630, -1.5630,  ..., -1.5805, -1.5805, -1.5805],
          [-1.5630, -1.5630, -1.5630,  ..., -1.5805, -1.5805, -1.5805],
          [-1.5630, -1.5630, -1.5630,  ..., -1.5805, -1.5805, -1.5805],
          ...,
          [-1.5805, -1.5805, -1.5805,  ..., -1.5630, -1.5630, -1.5630],
          [-1.5805, -1.5805, -1.5805,  ..., -1.5630, -1.5630, -1.5630],
          [-1.5805, -1.5805, -1.5805,  ..., -1.5630, -1.5630, -1.5630]],

         [[-1.3687, -1.3687, -1.3687,  ..., -1.3687, -1.3687, -1.3687],
          [-1.3687, -1.3687, -1.3687,  ..., -1.3687, -1.3687, -1.3687],
          [-1.3687, -1.3687, -1.3687,  ..., -1.3687, -1.3687, -1.3687],
          ...,
          [-1.3687, -1.3687, -1.3687,  ..., -1.3513, -1.3513, -1.3513],
          [-1.3687, -1.3687, -1.3687,  ..., -1.3513, -1.3513, -1.3513],
          [-1.3687, -1.3687, -1.3687,  ..., -1.3513, -1.3513, -1.3513]]],


        [[[-1.9809, -1.9809, -1.9809,  ..., -1.9467, -1.9467, -1.9467],
          [-1.9809, -1.9809, -1.9809,  ..., -1.9467, -1.9467, -1.9467],
          [-1.9809, -1.9809, -1.9809,  ..., -1.9467, -1.9467, -1.9467],
          ...,
          [-1.9295, -1.9295, -1.9295,  ..., -1.9295, -1.9295, -1.9295],
          [-1.9295, -1.9295, -1.9295,  ..., -1.9295, -1.9295, -1.9295],
          [-1.9295, -1.9295, -1.9295,  ..., -1.9295, -1.9295, -1.9295]],

         [[-1.5980, -1.5980, -1.5980,  ..., -1.5805, -1.5805, -1.5805],
          [-1.5980, -1.5980, -1.5980,  ..., -1.5805, -1.5805, -1.5805],
          [-1.5980, -1.5980, -1.5980,  ..., -1.5805, -1.5805, -1.5805],
          ...,
          [-1.5630, -1.5630, -1.5630,  ..., -1.5455, -1.5455, -1.5455],
          [-1.5630, -1.5630, -1.5630,  ..., -1.5455, -1.5455, -1.5455],
          [-1.5630, -1.5630, -1.5630,  ..., -1.5455, -1.5455, -1.5455]],

         [[-1.3861, -1.3861, -1.3861,  ..., -1.3687, -1.3687, -1.3687],
          [-1.3861, -1.3861, -1.3861,  ..., -1.3687, -1.3687, -1.3687],
          [-1.3861, -1.3861, -1.3861,  ..., -1.3687, -1.3687, -1.3687],
          ...,
          [-1.3513, -1.3513, -1.3513,  ..., -1.3339, -1.3339, -1.3339],
          [-1.3513, -1.3513, -1.3513,  ..., -1.3339, -1.3339, -1.3339],
          [-1.3513, -1.3513, -1.3513,  ..., -1.3339, -1.3339, -1.3339]]],


        [[[-1.9295, -1.9295, -1.9295,  ..., -1.9295, -1.9295, -1.9295],
          [-1.9295, -1.9295, -1.9295,  ..., -1.9295, -1.9295, -1.9295],
          [-1.9295, -1.9295, -1.9295,  ..., -1.9295, -1.9295, -1.9295],
          ...,
          [-1.9467, -1.9467, -1.9467,  ..., -1.9295, -1.9295, -1.9295],
          [-1.9467, -1.9467, -1.9467,  ..., -1.9295, -1.9295, -1.9295],
          [-1.9467, -1.9467, -1.9467,  ..., -1.9295, -1.9295, -1.9295]],

         [[-1.5805, -1.5805, -1.5805,  ..., -1.5630, -1.5630, -1.5630],
          [-1.5805, -1.5805, -1.5805,  ..., -1.5630, -1.5630, -1.5630],
          [-1.5805, -1.5805, -1.5805,  ..., -1.5630, -1.5630, -1.5630],
          ...,
          [-1.5630, -1.5630, -1.5630,  ..., -1.5630, -1.5630, -1.5630],
          [-1.5630, -1.5630, -1.5630,  ..., -1.5630, -1.5630, -1.5630],
          [-1.5630, -1.5630, -1.5630,  ..., -1.5630, -1.5630, -1.5630]],

         [[-1.3687, -1.3687, -1.3687,  ..., -1.3513, -1.3513, -1.3513],
          [-1.3687, -1.3687, -1.3687,  ..., -1.3513, -1.3513, -1.3513],
          [-1.3687, -1.3687, -1.3687,  ..., -1.3513, -1.3513, -1.3513],
          ...,
          [-1.3687, -1.3687, -1.3687,  ..., -1.3513, -1.3513, -1.3513],
          [-1.3687, -1.3687, -1.3687,  ..., -1.3513, -1.3513, -1.3513],
          [-1.3687, -1.3687, -1.3687,  ..., -1.3513, -1.3513, -1.3513]]]],
       device='cpu') tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], device='cpu')
3
Traceback (most recent call last):
  File "test_script.py", line 117, in <module>
    s.test_dir(device,model_ft,dataloaders)
  File "/gpfs/projects/LynchGroup/spacewhale/git_spacewhale/spacewhale/m_util2.py", line 198, in test_dir
    outputs = model_ft(im)
  File "/gpfs/projects/LynchGroup/spacewhale/space_env/lib/python3.7/site-packages/torch/nn/modules/module.py", line 477, in __call__
    result = self.forward(*input, **kwargs)
  File "/gpfs/projects/LynchGroup/spacewhale/space_env/lib/python3.7/site-packages/torch/nn/modules/container.py", line 91, in forward
    input = module(input)
  File "/gpfs/projects/LynchGroup/spacewhale/space_env/lib/python3.7/site-packages/torch/nn/modules/module.py", line 477, in __call__
    result = self.forward(*input, **kwargs)
  File "/gpfs/projects/LynchGroup/spacewhale/space_env/lib/python3.7/site-packages/torch/nn/modules/conv.py", line 301, in forward
    self.padding, self.dilation, self.groups)
RuntimeError: sizes must be non-negative
Mon May 20 20:49:45 EDT 2019
Now testing Resnext lr=0.2
/gpfs/projects/LynchGroup/spacewhale/git_spacewhale/spacewhale
Traceback (most recent call last):
  File "test_script.py", line 84, in <module>
    state_dict=torch.load(trained_model)
  File "/gpfs/projects/LynchGroup/spacewhale/space_env/lib/python3.7/site-packages/torch/serialization.py", line 356, in load
    f = open(f, 'rb')
FileNotFoundError: [Errno 2] No such file or directory: './trained_model/resnext_full256_lr2/epoch_23.pth'
Mon May 20 20:49:55 EDT 2019
