root@55cb7f729062:/workspace# ls
HF_HOME  Untitled.ipynb  __pycache__  dataset-cache  test.py  train_notebook_sdxl_mapping_saving.ipynb
root@55cb7f729062:/workspace# python
Python 3.10.12 (main, Jun 11 2023, 05:26:28) [GCC 11.4.0] on linux
Type "help", "copyright", "credits" or "license" for more information.
>>> from test import map_train
<class 'datasets.arrow_dataset.Dataset'>
You are using a model of type clip_text_model to instantiate a model of type . This is not supported for all configurations of models and can yield errors.
You are using a model of type clip_text_model to instantiate a model of type . This is not supported for all configurations of models and can yield errors.
>>> map_train()
Map (num_proc=2):   0%|                                                              | 0/833 [00:00<?, ? examples/s]<class 'datasets.arrow_dataset.Dataset'>
You are using a model of type clip_text_model to instantiate a model of type . This is not supported for all configurations of models and can yield errors.
You are using a model of type clip_text_model to instantiate a model of type . This is not supported for all configurations of models and can yield errors.
Process SpawnPoolWorker-1:
Traceback (most recent call last):
  File "/usr/local/lib/python3.10/dist-packages/multiprocess/process.py", line 314, in _bootstrap
    self.run()
  File "/usr/local/lib/python3.10/dist-packages/multiprocess/process.py", line 108, in run
    self._target(*self._args, **self._kwargs)
  File "/usr/local/lib/python3.10/dist-packages/multiprocess/pool.py", line 114, in worker
    task = get()
  File "/usr/local/lib/python3.10/dist-packages/multiprocess/queues.py", line 370, in get
    return _ForkingPickler.loads(res)
  File "/usr/local/lib/python3.10/dist-packages/dill/_dill.py", line 301, in loads
    return load(file, ignore, **kwds)
  File "/usr/local/lib/python3.10/dist-packages/dill/_dill.py", line 287, in load
    return Unpickler(file, ignore=ignore, **kwds).load()
  File "/usr/local/lib/python3.10/dist-packages/dill/_dill.py", line 442, in load
    obj = StockUnpickler.load(self)
  File "/usr/local/lib/python3.10/dist-packages/dill/_dill.py", line 432, in find_class
    return StockUnpickler.find_class(self, module, name)
  File "/workspace/test.py", line 220, in <module>
    set_start_method("spawn")
  File "/usr/local/lib/python3.10/dist-packages/multiprocess/context.py", line 247, in set_start_method
    raise RuntimeError('context has already been set')
RuntimeError: context has already been set
Map (num_proc=2):   0%|                                                              | 0/833 [00:19<?, ? examples/s]
Traceback (most recent call last):
  File "<stdin>", line 1, in <module>
  File "/workspace/test.py", line 224, in map_train
    return train_dataset.map(compute_embeddings_fn, batched=True, batch_size=4, with_rank=True, num_proc=2)
  File "/usr/local/lib/python3.10/dist-packages/datasets/arrow_dataset.py", line 592, in wrapper
    out: Union["Dataset", "DatasetDict"] = func(self, *args, **kwargs)
  File "/usr/local/lib/python3.10/dist-packages/datasets/arrow_dataset.py", line 557, in wrapper
    out: Union["Dataset", "DatasetDict"] = func(self, *args, **kwargs)
  File "/usr/local/lib/python3.10/dist-packages/datasets/arrow_dataset.py", line 3185, in map
    for rank, done, content in iflatmap_unordered(
  File "/usr/local/lib/python3.10/dist-packages/datasets/utils/py_utils.py", line 647, in iflatmap_unordered
    raise RuntimeError(
RuntimeError: One of the subprocesses has abruptly died during map operation.To debug the error, disable multiprocessing.
>>>