我们从Python开源项目中，提取了以下50个代码示例，用于说明如何使用wget.download()。
def run(argv): gencode_filepath = get_generated_path('sites/genes/gencode-{}.gtf.gz'.format(genes_version)) genes_filepath = common_filepaths['genes'] if not os.path.exists(genes_filepath): print('genes-{}.bed will be stored at {!r}'.format(genes_version, genes_filepath)) if not os.path.exists(gencode_filepath): make_basedir(gencode_filepath) wget.download( url="ftp://ftp.sanger.ac.uk/pub/gencode/Gencode_human/release_25/GRCh37_mapping/gencode.v25lift37.annotation.gtf.gz", out=gencode_filepath ) print('') genes = get_all_genes(gencode_filepath) genes = dedup_ensg(genes) genes = dedup_symbol(genes) make_basedir(genes_filepath) with open(genes_filepath, 'w') as f: writer = csv.DictWriter(f, delimiter='\t', fieldnames='chrom start end symbol ensg'.split(), lineterminator='\n') writer.writerows(genes) else: print("gencode is at {!r}".format(genes_filepath))
def download_content_file(self,dir_name,url): working_directory = os.getcwd() filename = None #if directory exists just catch error try: os.mkdir(dir_name) except: pass try: os.chdir(dir_name) filename = wget.download(url) except Exception as inst: print("had exception on wget: ", inst) pass #reset directory os.chdir(working_directory) return filename
def get_data(): """ Download the image,depth and segmentation data: Returns, the h5 database. """ if not osp.exists(DB_FNAME): try: colorprint(Color.BLUE,'\tdownloading data (56 M) from: '+DATA_URL,bold=True) print sys.stdout.flush() out_fname = 'data.tar.gz' wget.download(DATA_URL,out=out_fname) tar = tarfile.open(out_fname) tar.extractall() tar.close() os.remove(out_fname) colorprint(Color.BLUE,'\n\tdata saved at:'+DB_FNAME,bold=True) sys.stdout.flush() except: print colorize(Color.RED,'Data not found and have problems downloading.',bold=True) sys.stdout.flush() sys.exit(-1) # open the h5 file and return: return h5py.File(DB_FNAME,'r')
def configure(self, app_id, trigger, resources, dependencies): """ This method is meant to download the resources, resolve the dependencies, and prepare the app environment :param app_id: :param trigger: :param resources: :param dependencies: :return: """ instance = self.get_instance() instance.set_id(app_id) instance.set_command(trigger) self.create_env(instance) self.get_resources(instance, resources) self.resolve_dependencies(dependencies) return instance #def start(self, instance): # self.spawn(instance.get_command()) # return instance
def download_file(self, url, path): """ logic that does all the work when downloading the resources :param url: :param path: :return: """ file_name = url.split("/")[-1] file_location = path + "/" + file_name try: os.remove(file_location) except OSError: pass wget.download(url, out=file_location, bar=None) return file_location
def Downloadfiles(urls_metadata,output,target): path = None try: filename = None print "\nDo you like downloading these files to analyze metadata(Y/N)?" #Convert to lower the input resp = raw_input().lower() if (resp == 'n'): print "Exiting" exit(1) if ((resp != 'y') and (resp != 'n')): print "The option is not valided. Please, try again it" if (resp =='y'): path = str(target) + '/temp' for url in urls_metadata: filename = wget.download(url,str(path)) Analyze_Metadata(filename) time.sleep(3) #Delete temp folder os.system('rm -r ' + str(path)) Display_Export_Metadata(metadata_files,output,target) except Exception as e: print str(e)
def Downloadfiles(urls_metadata): print "\nDo you like downloading these files to analyze metadata(Y/N)?" #try: resp = raw_input() if (resp == 'N'): print "Exiting" exit(1) if ((resp != 'Y') and (resp != 'N')): print "The option is not valided. Please, try again it" if (resp =='Y'): print "Indicate the location where keep the files downloaded" path = raw_input() try: for url in urls_metadata: try: filename = wget.download(url,path) Analyze_Metadata(filename) except: pass Display_Export_Metadata(metadata_files) except: pass #********************************************************# #Definition and treatment of the parameters
def Downloadfiles(urls_metadata): print "\nDo you like downloading these files to analyze metadata(Y/N)?" #try: resp = raw_input() if (resp == 'N'): print "Exiting" exit(1) if ((resp != 'Y') and (resp != 'N')): print "The option is not valided. Please, try again it" if (resp =='Y'): try: for url in urls_metadata: try: filename= wget.download(url,"/opt/") Analyze_Metadata(filename) except Exception, e: print e except: pass #********************************************************# #Definition and treatment of the parameters
def Downloadfiles(urls_metadata): print "\nDo you like downloading these files to analyze metadata(Y/N)?" #try: resp = raw_input().lower() if (resp == 'n'): print "Exiting" exit(1) if ((resp != 'y') and (resp != 'n')): print "The option is not valided. Please, try again it" if (resp =='y'): print "Indicate the location where you want to keep the files downloaded" path = raw_input() try: for url in urls_metadata: try: filename= wget.download(url,path) Analyze_Metadata(filename) except Exception, e: print e except: pass #********************************************************# #Definition and treatment of the parameters
def main(): root_path = 'an4/' name = 'an4' wget.download('http://www.speech.cs.cmu.edu/databases/an4/an4_raw.bigendian.tar.gz') tar = tarfile.open('an4_raw.bigendian.tar.gz') tar.extractall() os.makedirs(args.target_dir) _format_data(root_path, 'train', name, 'an4_clstk') _format_data(root_path, 'test', name, 'an4test_clstk') shutil.rmtree(root_path) os.remove('an4_raw.bigendian.tar.gz') train_path = args.target_dir + '/train/' test_path = args.target_dir + '/test/' print ('\n', 'Creating manifests...') create_manifest(train_path, 'an4_train') create_manifest(test_path, 'an4_val')
def download_figshare(file_name, file_ext, dir_path='./', change_name = None): prepare_data_dir(dir_path) url = 'https://ndownloader.figshare.com/files/' + file_name wget.download(url, out=dir_path) file_path = os.path.join(dir_path, file_name) if file_ext == '.zip': zip_ref = zipfile.ZipFile(file_path,'r') if change_name is not None: dir_path = os.path.join(dir_path, change_name) zip_ref.extractall(dir_path) zip_ref.close() os.remove(file_path) elif file_ext == '.tar.bz2': tar_ref = tarfile.open(file_path,'r:bz2') if change_name is not None: dir_path = os.path.join(dir_path, change_name) tar_ref.extractall(dir_path) tar_ref.close() os.remove(file_path) elif change_name is not None: os.rename(file_path, os.path.join(dir_path, change_name)) # Download QM9 dataset
def run(argv): if not os.path.exists(clean_filepath): print('dbsnp will be stored at {!r}'.format(clean_filepath)) if not os.path.exists(raw_filepath): # dbSNP downloads are described at <https://www.ncbi.nlm.nih.gov/variation/docs/human_variation_vcf/> # This file includes chr-pos-ref-alt-rsid and 4X a bunch of useless columns: dbsnp_url = 'ftp://ftp.ncbi.nlm.nih.gov/snp/organisms/human_9606_b150_GRCh37p13/VCF/00-All.vcf.gz' print('Downloading dbsnp!') make_basedir(raw_filepath) raw_tmp_filepath = get_tmp_path(raw_filepath) wget.download(url=dbsnp_url, out=raw_tmp_filepath) print('') os.rename(raw_tmp_filepath, raw_filepath) print('Done downloading.') print('Converting {} -> {}'.format(raw_filepath, clean_filepath)) make_basedir(clean_filepath) clean_tmp_filepath = get_tmp_path(clean_filepath) run_script(r''' gzip -cd '{raw_filepath}' | grep -v '^#' | perl -F'\t' -nale 'print "$F[0]\t$F[1]\t$F[2]\t$F[3]\t$F[4]"' | # Gotta declare that it's tab-delimited, else it's '\s+'-delimited I think. gzip > '{clean_tmp_filepath}' '''.format(raw_filepath=raw_filepath, clean_tmp_filepath=clean_tmp_filepath)) os.rename(clean_tmp_filepath, clean_filepath) print("dbsnp is at '{clean_filepath}'".format(clean_filepath=clean_filepath))
def download_ref(build, chrom): '''Download a chromosome reference file, and remove the header and newlines so we can seek to positions.''' dest_filepath = ref_filepath(build, chrom, download=False) if os.path.exists(dest_filepath): return dl_filepath = get_tmp_path('dl-chrom-{}-{}'.format(build['hg'], chrom)) if not os.path.exists(dl_filepath): dl_tmp_filepath = get_tmp_path(dl_filepath) url = 'ftp://hgdownload.cse.ucsc.edu/goldenPath/{}/chromosomes/chr{}.fa.gz'.format(build['hg'], chrom) wget.download(url=url, out=dl_tmp_filepath) print('') os.rename(dl_tmp_filepath, dl_filepath) tmp_filepath = get_tmp_path(dest_filepath) run_script(r''' gzip -cd '{dl_filepath}' | tail -n +2 | tr -d "\n" > '{tmp_filepath}' '''.format(dl_filepath=dl_filepath, tmp_filepath=tmp_filepath)) os.rename(tmp_filepath, dest_filepath) print("ref is at", dest_filepath)
def ref_filepath(build, chrom, download=True): filepath = get_cacheable_file_location('ref', 'reference-{}-chrom-{}.fa'.format(build['hg'], chrom)) if download: download_ref(build, chrom) return filepath
def download_ckpt(url): fname = wget.download(url) with tarfile.open(fname) as tar: tar.extractall() ckpt_name = tar.getnames()[0] if not os.path.exists(FLAGS.checkpoints_dir): os.makedirs(FLAGS.checkpoints_dir) shutil.move(ckpt_name, FLAGS.checkpoints_dir) os.remove(fname)
def download_file(url, directory='./', local_fname=None, force_write=False): """Download the data from source url, unless it's already here. Args: filename: string, name of the file in the directory. work_directory: string, path to working directory. source_url: url to download from if file doesn't exist. Returns: Path to resulting file. """ if not os.path.isdir(directory): os.mkdir(directory) if local_fname is None: local_fname = url.split('/')[-1] local_fname = os.path.join(directory, local_fname) if os.path.exists(local_fname) and not force_write: print ("File [{}] existed!".format(local_fname)) return local_fname print ("Downloading file [{}] from [{}]".format(local_fname, url)) try: import wget return wget.download(url, local_fname) except: return _single_thread_download(url, local_fname)
def download_to(url, path_to_dir): """ Downloads a file from a given to a target directory. Returns the file name if the downloaded file. """ logger.debug("downloading file from %s to %s" % (url, path_to_dir)) filename = wget.download(url, out=path_to_dir, bar=None) return filename
def run(self): """Runs install. """ self.initOptions() #Try to download gmsh if self.dGmsh: self.downloadGmsh() else: self.gmshDownloaded=False #Try to download fiji if self.dFiji: self.downloadFiji() else: self.fijiDownloaded=False #Run setuptools install install.run(self) #Print log info log.info("Overriding setuptools mode of scripts ...") #Add Data and edit file permissions self.addData()
def downloadFileIfNotExist(self,url): """Downloads URL if file does not already exist. Args: url (str): URL to download. Returns: tuple: Tuple containing: * fnDL (str): Donwload filename * folderFn (str): Filename of extracted download files """ import wget cwd=os.getcwd()+"/" if not os.path.exists(cwd+os.path.basename(url)): print cwd+os.path.basename(url) +" does not exist, will download it." folderFn=wget.download(url) else: print cwd+os.path.basename(url) +" alreay exists, will not download." folderFn=os.path.basename(url) fnDL=str(folderFn) print return folderFn, fnDL
def downloadGmshWin(self,arch,gmshVersion): """Downloads Gmsh from Gmsh website for Windows Args: arch (str): System architecture, e.g. 64/32. gmshVersion (str): gmshVersion String, e.g. 2.12.0 . Returns: tuple: Tuple containing: * fnDL (str): Donwload filename * folderFn (str): Filename of extracted download files """ #Download Gmsh url='http://gmsh.info/bin/Windows/gmsh-'+gmshVersion+'-Windows'+arch+'.zip' folderFn, fnDL=self.downloadFileIfNotExist(url) #Decompress import zipfile with zipfile.ZipFile(folderFn) as zf: zf.extractall() folderFn='gmsh-'+gmshVersion+'-Windows' self.gmshPath='executables/gmsh/bin/gmsh.exe' return fnDL,folderFn
def downloadFijiLinux(self,arch): """Downloads Fiji from Fiji website for Linux. Args: arch (str): System architecture, e.g. 64/32. Returns: tuple: Tuple containing: * fnDL (str): Donwload filename * folderFn (str): Filename of extracted download files """ import wget #Download Fiji url='http://downloads.imagej.net/fiji/latest/fiji-linux'+arch+'.zip' folderFn=wget.download(url) fnDL=str(folderFn) print #Decompress import zipfile with zipfile.ZipFile(folderFn) as zf: zf.extractall() folderFn='Fiji.app' self.fijiPath='executables/Fiji.app/./ImageJ-linux64' return fnDL,folderFn
def downloadFijiWin(self,arch): """Downloads Fiji from Fiji website for Windows. Args: arch (str): System architecture, e.g. 64/32. Returns: tuple: Tuple containing: * fnDL (str): Donwload filename * folderFn (str): Filename of extracted download files """ import wget #Download fiji url='http://downloads.imagej.net/fiji/latest/fiji-win'+arch+'.zip' folderFn=wget.download(url) fnDL=str(folderFn) print #Decompress import zipfile with zipfile.ZipFile(folderFn) as zf: zf.extractall() folderFn='Fiji.app' self.fijiPath='executables/Fiji.app/ImageJ-linux64.exe' return fnDL,folderFn
def downloadFijiOSX(self,arch): """Downloads Fiji from Fiji website for OSX. Returns: tuple: Tuple containing: * fnDL (str): Donwload filename * folderFn (str): Filename of extracted download files """ import wget #Download fiji url='http://downloads.imagej.net/fiji/latest/fiji-macosx.dmg' folderFn=wget.download(url) fnDL=str(folderFn) print #Mount dmg file os.system('hdiutil attach '+folderFn) cwd=os.getcwd() #Copy fiji executable to cwd os.system('cp -rv /Volumes/Fiji/Fiji.app '+ cwd) #Unmount gmsh os.system('hdiutil detach /Volumes/Fiji') folderFn='Fiji.app' self.fijiPath='executables/Fiji.app/Contents/MacOS/./ImageJ-macosx' return fnDL,folderFn
def add_arguments(self, parser): parser.add_argument("--reset", dest="reset", action="store_true", help="If true, clear the list of images and download all of them again")
def ShowResults(newlist,num_files,target): print "Files in the target "+target+" are:\n" print "Files indexed:", len (urls_final) for i in urls_final: if i not in newlist: newlist.append(i) print i #verify if the user wants to export results if output == 'Y': #Only it can enter if -j is put in the execution ExportResults(newlist) #Call to function to download the files Downloadfiles(newlist) #MAIN
def ShowResults(newlist,num_files,target): print "Files in the target "+target+" are:\n" print "Files indexed:", len (urls_final) for i in urls_final: if i not in newlist: newlist.append(i) print i #verify if the user wants to export results if output == 'Y': #Only it can enter if -j is put in the execution ExportResults(newlist) #Call to function to download the files Downloadfiles(newlist) #INICIO MAIN
def Downloadfiles(urls_metadata,output): try: print "\nDo you like downloading these files to analyze metadata(Y/N)?" #Convert to lower the input resp = raw_input().lower() if (resp == 'n'): print "Exiting" exit(1) if ((resp != 'y') and (resp != 'n')): print "The option is not valided. Please, try again it" if (resp =='y'): print "Indicate the location where you want to keep the files downloaded", path = raw_input() try: for url in urls_metadata: try: filename = wget.download(url,path) Analyze_Metadata(filename) except: pass Display_Export_Metadata(metadata_files,output) except: pass except Exception as e: print str(e) #********************************************************# #Definition and treatment of the parameters
def ShowResults(newlist,num_files,target,output): print "Files in the target "+target+" are:\n" print "Files indexed:", len (urls_final) for i in urls_final: if i not in newlist: newlist.append(i) print i #verify if the user wants to export results if output == 'Y': #Only it can enter if -j is put in the execution ExportResults(newlist) #Call to function to download the files Downloadfiles(newlist,output) #MAIN
def ShowResults(newlist,num_files,target,output,option,captcha): if option == 1: print "Files into the target "+target+" are:\n" if captcha == False: print "Files indexed:", len (url_google) for i in url_google: #if i not in url_google: newlist.append(i) print i else: ##Catpcha == True, try results of Bing print "Files indexed:", len (urls_final) for i in urls_final: #if i not in url_google: newlist.append(i) print i else: #option ==2 print "ShowResults outside target" print "Files outside target "+target+" are:\n" print "Files indexed:", len (url_google) for i in url_google: #if i not in url_google: newlist.append(i) print i #verify if the user wants to export results if (output == 'Y') or (output =='y'): #Only it can enter if -j is put in the execution ExportResults(newlist) #Call to function to download the files Downloadfiles(newlist,output) #MAIN
def Downloadfiles(urls_metadata,output,target): path = None try: filename = None print "\nDo you like downloading these files to analyze metadata(Y/N)?" #Convert to lower the input resp = raw_input().lower() if (resp == 'n'): print "Exiting" exit(1) if ((resp != 'y') and (resp != 'n')): print "The option is not valided. Please, try again it" if (resp =='y'): print "Indicate the location where you want to keep the files downloaded.by default in the target folder", path = raw_input() #path = createdir.CreateDir(target) for url in urls_metadata: filename = wget.download(url,str(path)) Analyze_Metadata(filename) Display_Export_Metadata(metadata_files,output,target) if count_pdf > 1: os.system('mv *pdf '+ str(target)) if count_word > 1: os.system('mv *doc '+ str(target)) os.system('mv *docx '+ str(target)) if count_others > 1: os.system('mv *xlsx '+ str(target)) os.system('mv *ppt '+ str(target)) except Exception as e: print str(e)
def load(self): """ Load a dataset. If a dataset has not been downloaded yet, this class downloads it. """ # download a dataset if needed. self._download() # load joints. self.joints = self._load_joints()
def _download(self): # download when file doesn't exist. path = os.path.join(self.path, self.name) if not os.path.isdir(path): # download a dataset. path = wget.download(self.url, self.path) # extract the dataset. with zipfile.ZipFile(path, 'r') as zip_file: zip_file.extractall(self._get_extract_path()) # remove downloaded zip file. os.remove(path)
def upload_remote_image(self, cdn_path, url): """Upload remote image to CDN. Args: cdn_path: the name of file storing in CDN url: the remote image URL Returns: return whether it's uploaded. """ file_path = wget.download(url) result = self.upload_local_image(cdn_path, file_path) os.remove(file_path) return result
def maybe_download_model(filename='ResNet_18.model'): if(os.path.isfile(filename)): print('Model %s already downloaded' % filename) else: model_name_to_url = { 'ResNet_18.model': 'http://migonzastorage.blob.core.windows.net/deep-learning/models/cntk/rc3/ResNet_18.model', 'ResNet_152.model': 'http://migonzastorage.blob.core.windows.net/deep-learning/models/cntk/rc3/ResNet_152.model' } url = model_name_to_url[filename] wget.download(url)
def download_dstc2_data(): if not os.path.isfile('./tmp/dstc2_traindev.tar.gz'): wget.download('http://camdial.org/~mh521/dstc/downloads/dstc2_traindev.tar.gz', './tmp') wget.download('http://camdial.org/~mh521/dstc/downloads/dstc2_test.tar.gz', './tmp') wget.download('http://camdial.org/~mh521/dstc/downloads/dstc2_scripts.tar.gz', './tmp') print() else: print('Everything appears to be downloaded.')
def download(files, delay=0, prefix=file_prefix, convert=False): # Given a list of files, downloads each, optionally also converts to csv total = len(files) i = 1 for file in files: filename = prefix+file wget.download(filename) print("Downloaded", i, "files out of", total) i += 1 # In case the webpage doesn't like many rapid downloads time.sleep(delay)
def download_files(tag): paths_in_website = searchfiles(tags_url, host, tag, search) [wget.download(path_in_website, out = tag) for path_in_website in paths_in_website]
def download_files(): paths_in_website = searchfiles() [wget.download(path_in_website, out = "archive") for path_in_website in paths_in_website]
