批量从NCBI后台下载指定数据的Perl脚本

??????? 最近需要在NCBI中下载所有Xanthomonas属菌株对应的gbk文件，由于NCBI前台gbk数据已经改版，故打算从后台ftp.ncbi.nlm.nih.gov下载。写了个Perl脚本用于批量下载NCBI后台数据，有这方面需求的同仁们可以参考。另外，多进程暂时未成功，后期再更改。

#!/usr/bin/perl
##################################################################################################
# NCBI_ftp_batch_fetch.pl
# 黄良博 huanglb0805@163.com
# 2015-3-25
# 用途：可用于在NCBI的后台ftp.ncbi.nlm.nih.gov/genomes/Bacteria中下载某一个属对应的文件（gbk,fna,gbs,ptt等均可）。
# 下载Xanthomonas属中所有菌株的gbk文件和fna文件：
# perl $0 -g Xanthomonas -s gbk -s fna
# 注意：多进程暂时不能使用
##################################################################################################

use v5.16;
use warnings;
use Net::FTP;
use Cwd;
use Getopt::Long;
use Parallel::ForkManager;

my($genus,@suffix,$thread,$help);

GetOptions(
	"genus:s"	=>$genus,"suffix:s"	=>@suffix,#"thread:i"	=>$thread,"help"		=>$help,);
#如果命令行上有多余参数，则报错并退出
if (@ARGV!=0){
	say 'Please check your parameters,maybe you forgot to add "" or _ to genus';
	exit;
}
	
my $usage=<<USAGE;
Please enter parameters!
-help		help
-genus		genus
-suffix		file type(eg. gbk,ptt,...)
-thread		number of thread
Usage:
  perl $0 -g genus -s suffix
eg.
  perl $0 -g Xanthomonas -s gbk -s fna
USAGE

if ($help or not defined $genus or @suffix==0){
    say "$usage";
    exit();
}

#----------------------------------------
my $host="ftp.ncbi.nlm.nih.gov";
my $directory="/genomes/Bacteria";

my ($newerr,$ftp,@files);

$ftp=Net::FTP->new($host,Timeout=>240) 
	or die "Can't ftp to $host: $!n";

say "Connected";

$ftp->login("ftp","apl@") 
	or die "Can't login to $host: $!n";
	
say "Getting file list...";

$ftp->cwd($directory) 
	or die "Can't cd  $!n";

#建立下载目录
my $restore_dir = "C:/Users/liangbo/Desktop/download";	#设定下载路径
unless (-d $restore_dir){
    mkdir $restore_dir,0755 or die "Cannot make directory: $!";
}
chdir $restore_dir;

@files=$ftp->dir
	or die "Can't get file list $!n";

say "Got file list";

#如路径为Xanthomonas_oryzae_KACC_10331_uid58155
#利用qr在编译时内插，提升性能
$genus=~s/s+/_/g;
my $regex=qr/($genus.*)/;

my %count;	#统计信息
my $count_strains=0;
my ($filename,$path);

$thread||=0;	#默认为单进程
my $pm=Parallel::ForkManager->new($thread);

for(@files){
	$pm->start and next;
	if(/$regex/){
		$count_strains++;	#统计找到的菌株数目
		for my $ext (@suffix){
			my $retrive="/genomes/Bacteria/$1/*.$ext";
			my @file=$ftp->dir($retrive);
			for (@file){
				if(/(/genomes/Bacteria/(.*.$ext))/){
					($filename,$path)=($2,$1);
					say $filename;	#输出找到的菌株
					$count{$ext}{found}++;	#统计找到的文件数目
					#download,注意此处对应的路径为/genomes/Bacteria/Xanthomonas_.../*.gbk
					#将形如Xanthomonas_oryzae_KACC_10331_uid58155/NC_006834.gbk文件名改为
					#Xanthomonas_oryzae_KACC_10331_uid58155_NC_006834.gbk,并以此命名
					$filename=~s///_/g;
					$ftp->get($path,$filename) 
						or die "get failed ",$ftp->message;
					$count{$ext}{got}++;	#统计成功下载的文件数目
				}
			}
		}
	}
	$pm->finish;
}

$pm->wait_all_children;
$ftp->quit;

my $cwd=getcwd();

#输出统计信息
if($count_strains){
	say "Found:";
	say "Strains: $count_strains";
	for my $ext (keys %count){
		say "$ext files:";
		for my $item (sort keys %{$count{$ext}}){
			say "t$itemt$count{$ext}{$item}";
		}
	}
	say "Files downloaded in $cwd";
}else{
	say "Attention: Nothing found,check your genus input!";
}