Add some random to temporary small indices path (#155)

In order to avoid failures upon concurrent executions.
Only used when `distributed = "pyspark"`

Co-authored-by: Quentin Auge <q.auge@criteo.com>
Co-authored-by: Romain Beaumont <r.beaumont@criteo.com>

autofaiss/indices/distributed.py

@@ -10,6 +10,7 @@
 from tempfile import TemporaryDirectory
 import tempfile
 from typing import Dict, Optional, Iterator, Tuple, Callable, Any, Union, List
+import uuid
 from functools import partial
 from multiprocessing.pool import ThreadPool
 
@@ -285,6 +286,9 @@ def add_embeddings_to_index_distributed(
     index_optimizer: Optional[Callable]
         The function that optimizes the index
     """
+    temporary_indices_folder = temporary_indices_folder.rstrip("/") + f"/{uuid.uuid4().hex}"
+    logger.debug(f"Will be writing temporary small indices to {temporary_indices_folder}")
+
     temporary_indices_folder = make_path_absolute(temporary_indices_folder)
     fs = _get_file_system(temporary_indices_folder)
     if fs.exists(temporary_indices_folder):