9jaddlZddlZddlmZmZddlmZddlmZddl m Z ddl m Z ddl mZddlmZmZmZmZe rdd lmZd d gZGd d eZGdd eZGdd eZy)N) GeneratorSequence)contextmanager)Path) TYPE_CHECKING) url_to_fs)StreamTransformExtension)FileSystemBaseFileSystemReaderFileSystemWriterSerializationFormat)AbstractFileSystem FsspecWriter FsspecReaderc "eZdZddZedeejzdedee jddffdZ deejzdedeejzfdZ deejzdeejzfd Z deejzd eejzddfd Zdeejzddfd Zed eejzdefdZdeejzdefdZdeejzddfdZdeejzdeefdZy) FileSystemreturnNcd|_yN)fs)selfs o/media/conek/DATA/Code/OCR/venv/lib/python3.12/site-packages/torch/distributed/checkpoint/_fsspec_filesystem.py__init__zFileSystem.__init__!s -1pathmodec#<K|j tdtj|}|jj |5} | dddy#t fddDr |j |#YxYwxYw#1swYyxYww)Nzfs should not be Nonec3&K|]}|v ywr).0chrs r z+FileSystem.create_stream..3s2brTz2szw+a)rAssertionErrorosfspathopenanyrm_file)rrrstreams ` r create_streamzFileSystem.create_stream$s 77? !89 9yy WW\\$ %      2E22 T*  sHA B BA BB 3BB B B  BBBsuffixcBtjj||Sr)r$rjoin)rrr+s r concat_pathzFileSystem.concat_path:sww||D&))rc .t|fi|\|_}|Sr)rr)rrkwargs_s r init_pathzFileSystem.init_path=st.v.  rnew_pathc<|jj||yr)rrename)rrr3s rr5zFileSystem.renameAs tX&rc>|jj|dy)NT)exist_ok)rmakedirsrrs rmkdirzFileSystem.mkdirDs -r checkpoint_idc\t|try t|y#t$rYywxYw)NFT) isinstancerr ValueErrorclsr;s rvalidate_checkpoint_idz!FileSystem.validate_checkpoint_idGs5 mT *  m $  s  ++c8|jj|Sr)rexistsr9s rrCzFileSystem.existsSsww~~d##rc:|jj|yr)rrmr9s rr(zFileSystem.rm_fileVs  4rc<|jj|dS)NF)detail)rlsr9s rrHz FileSystem.lsYswwzz$uz--r)rN)__name__ __module__ __qualname__rrstrr$PathLikerioIOBaser*r.r2r5r: classmethodboolrArCr(listrHrrrrr sh2"++%-0 299dD( )**bkk 1*3*3CT*cBKK/cBKK>O'3,'bkk8I'd'.# +.. 33D   $3,$$C"++-$.sR[[(.T#Y.rrceZdZdZddddddej fdeejzde de d e d e d e d e e dzd eddffd Z edeejzde fdZxZS)ra` Basic implementation of StorageWriter using FFspec. This implementation makes the following assumptions and simplifications: * The checkpoint path is an empty or non-existing directory. * File creation is atomic The checkpoint consist of one file per write request plus a `.metadata` file with the serialized metadata. TiNrsingle_file_per_rank sync_files thread_countper_thread_copy_ahead overwrite _extensionsserialization_formatrc t |||||||||t|_|jj|fi| |_y)a= Initialize the writer pointing to `path`. Args: path: directory where the checkpoint will be written to. single_file_per_rank: Produce one file per rank instead of one file per tensor/blob. Default to True. sync_files : force files to be synced to permanent storage. Default to True. thread_count: Number of IO threads to use to write. Default to 1. per_thread_copy_ahead: How many bytes to copy from the GPU ahead of saving then. Default 10Mb. overwrite: Whether to allow overwriting existing checkpoints. Defaults to True. _extensions: Extensions to apply to output streams (EXPERIMENTAL) N. B. If sync_files is disabled, there's no guarantee that the checkpoint will be consistent in the case of a failure. )rYrZr[Nsuperrrrr2r) rrrUrVrWrXrYrZr[r0 __class__s rrzFsspecWriter.__init__nsW4     !#!5  ,%DGG%%d5f5 rr;c,tj|SrrrAr?s rrAz#FsspecWriter.validate_checkpoint_id00??r)rIrJrK__doc__r TORCH_SAVErLr$rMrQintrr rrPrA __classcell__r_s@rrr`s  &*%/AE4G4R4R%6BKK%6#%6 %6  %6 # %6%667$>%62%6 %6N@33D@@@rcveZdZdeej zddffd Zedeej zdefdZ xZ S)rrrNc t||t|_|jj|fi||_yrr])rrr0r_s rrzFsspecReader.__init__s6 ,%DGG%%d5f5 rr;c,tj|Srrar?s rrAz#FsspecReader.validate_checkpoint_idrbr) rIrJrKrLr$rMrrPrQrArfrgs@rrrsN6S2;;.6T6 @33D@@@r)rNr$collections.abcrr contextlibrpathlibrtypingr fsspec.corer'torch.distributed.checkpoint._extensionr 'torch.distributed.checkpoint.filesystemr r r r fsspecr__all__rrrrrrrtsl /% !L)  <.<.@7@#7@t@#@r