Datasets: Fail to download c4 english corpus

[donggyukimc]

i run following code to download c4 English corpus.

dataset = nlp.load_dataset('c4', 'en', beam_runner='DirectRunner'
, data_dir='/mypath')

and i met failure as follows

Downloading and preparing dataset c4/en (download: Unknown size, generated: Unknown size, total: Unknown size) to /home/adam/.cache/huggingface/datasets/c4/en/2.3.0...
Traceback (most recent call last):
  File "download_corpus.py", line 38, in <module>
    , data_dir='/home/adam/data/corpus/en/c4')
  File "/home/adam/anaconda3/envs/adam/lib/python3.7/site-packages/nlp/load.py", line 520, in load_dataset
    save_infos=save_infos,
  File "/home/adam/anaconda3/envs/adam/lib/python3.7/site-packages/nlp/builder.py", line 420, in download_and_prepare
    dl_manager=dl_manager, verify_infos=verify_infos, **download_and_prepare_kwargs
  File "/home/adam/anaconda3/envs/adam/lib/python3.7/site-packages/nlp/builder.py", line 816, in _download_and_prepare
    dl_manager, verify_infos=False, pipeline=pipeline,
  File "/home/adam/anaconda3/envs/adam/lib/python3.7/site-packages/nlp/builder.py", line 457, in _download_and_prepare
    split_generators = self._split_generators(dl_manager, **split_generators_kwargs)
  File "/home/adam/anaconda3/envs/adam/lib/python3.7/site-packages/nlp/datasets/c4/f545de9f63300d8d02a6795e2eb34e140c47e62a803f572ac5599e170ee66ecc/c4.py", line 175, in _split_generators
    dl_manager.download_checksums(_CHECKSUMS_URL)
AttributeError: 'DownloadManager' object has no attribute 'download_checksums

can i get any advice?

[lhoestq]

Hello ! Thanks for noticing this bug, let me fix that.

Also for information, as specified in the changelog of the latest release, C4 currently needs to have a runtime for apache beam to work on. Apache beam is used to process this very big dataset and it can work on dataflow, spark, flink, apex, etc. You can find more info on beam datasets here.

Our goal in the future is to make available an already-processed version of C4 (as we do for wikipedia for example) so that users without apache beam runtimes can load it.

[prashant-kikani]

@lhoestq I am facing IsADirectoryError while downloading with this command.
Can you pls look into it & help me.
I'm using version 0.4.0 of nlp.

dataset = load_dataset("c4", 'en', data_dir='.', beam_runner='DirectRunner')

Here's the complete stack trace.

Downloading and preparing dataset c4/en (download: Unknown size, generated: Unknown size, post-processed: Unknown sizetotal: Unknown size) to /home/devops/.cache/huggingface/datasets/c4/en/2.3.0/096df5a27756d51957c959a2499453e60a08154971fceb017bbb29f54b11bef7...

---------------------------------------------------------------------------
IsADirectoryError                         Traceback (most recent call last)
<ipython-input-11-f622e6705e03> in <module>
----> 1 dataset = load_dataset("c4", 'en', data_dir='.', beam_runner='DirectRunner')

/data/anaconda/envs/hf/lib/python3.6/site-packages/nlp/load.py in load_dataset(path, name, version, data_dir, data_files, split, cache_dir, features, download_config, download_mode, ignore_verifications, save_infos, **config_kwargs)
    547     # Download and prepare data
    548     builder_instance.download_and_prepare(
--> 549         download_config=download_config, download_mode=download_mode, ignore_verifications=ignore_verifications,
    550     )
    551 

/data/anaconda/envs/hf/lib/python3.6/site-packages/nlp/builder.py in download_and_prepare(self, download_config, download_mode, ignore_verifications, try_from_hf_gcs, dl_manager, **download_and_prepare_kwargs)
    461                 if not downloaded_from_gcs:
    462                     self._download_and_prepare(
--> 463                         dl_manager=dl_manager, verify_infos=verify_infos, **download_and_prepare_kwargs
    464                     )
    465                 # Sync info

/data/anaconda/envs/hf/lib/python3.6/site-packages/nlp/builder.py in _download_and_prepare(self, dl_manager, verify_infos)
    964         pipeline = beam_utils.BeamPipeline(runner=beam_runner, options=beam_options,)
    965         super(BeamBasedBuilder, self)._download_and_prepare(
--> 966             dl_manager, verify_infos=False, pipeline=pipeline,
    967         )  # TODO handle verify_infos in beam datasets
    968         # Run pipeline

/data/anaconda/envs/hf/lib/python3.6/site-packages/nlp/builder.py in _download_and_prepare(self, dl_manager, verify_infos, **prepare_split_kwargs)
    516         split_dict = SplitDict(dataset_name=self.name)
    517         split_generators_kwargs = self._make_split_generators_kwargs(prepare_split_kwargs)
--> 518         split_generators = self._split_generators(dl_manager, **split_generators_kwargs)
    519         # Checksums verification
    520         if verify_infos:

/data/anaconda/envs/hf/lib/python3.6/site-packages/nlp/datasets/c4/096df5a27756d51957c959a2499453e60a08154971fceb017bbb29f54b11bef7/c4.py in _split_generators(self, dl_manager, pipeline)
    187         if self.config.realnewslike:
    188             files_to_download["realnews_domains"] = _REALNEWS_DOMAINS_URL
--> 189         file_paths = dl_manager.download_and_extract(files_to_download)
    190 
    191         if self.config.webtextlike:

/data/anaconda/envs/hf/lib/python3.6/site-packages/nlp/utils/download_manager.py in download_and_extract(self, url_or_urls)
    218             extracted_path(s): `str`, extracted paths of given URL(s).
    219         """
--> 220         return self.extract(self.download(url_or_urls))
    221 
    222     def get_recorded_sizes_checksums(self):

/data/anaconda/envs/hf/lib/python3.6/site-packages/nlp/utils/download_manager.py in download(self, url_or_urls)
    156             lambda url: cached_path(url, download_config=self._download_config,), url_or_urls,
    157         )
--> 158         self._record_sizes_checksums(url_or_urls, downloaded_path_or_paths)
    159         return downloaded_path_or_paths
    160 

/data/anaconda/envs/hf/lib/python3.6/site-packages/nlp/utils/download_manager.py in _record_sizes_checksums(self, url_or_urls, downloaded_path_or_paths)
    106         flattened_downloaded_path_or_paths = flatten_nested(downloaded_path_or_paths)
    107         for url, path in zip(flattened_urls_or_urls, flattened_downloaded_path_or_paths):
--> 108             self._recorded_sizes_checksums[url] = get_size_checksum_dict(path)
    109 
    110     def download_custom(self, url_or_urls, custom_download):

/data/anaconda/envs/hf/lib/python3.6/site-packages/nlp/utils/info_utils.py in get_size_checksum_dict(path)
     77     """Compute the file size and the sha256 checksum of a file"""
     78     m = sha256()
---> 79     with open(path, "rb") as f:
     80         for chunk in iter(lambda: f.read(1 << 20), b""):
     81             m.update(chunk)

IsADirectoryError: [Errno 21] Is a directory: '/'

Can anyone please try to see what I am doing wrong or is this a bug?

[AlexRothebest]

I have the same problem as @prashant-kikani

[lhoestq]

Looks like a bug in the dataset script, can you open an issue ?

[fredriko]

I see the same issue as @prashant-kikani. I'm using datasets version 1.2.0 to download C4.