Sun Grid Engine 大规模集群监控
#!/usr/bin/perl
#!/bin/bash
## 最近查看队列使用情况 发现如下问题，用户使用SGE 集群的时候内存溢出
## 此程序用于查看SGE (Sun Grid Engine) 整体集群监控
##仅以此程序，帮助大家查看 自己任务状态，以免被杀！此工具归BGI所有，祝大家工作愉快！
=head1 Name
sge_cluster_queue.pl -- choose the queue observation
=head1 Description
This program can choose the bmk queue observation
1) look for the queue jobs operation and who is working state
2) find the jobs number total test number,and the user over mem jobs,Jobs-ID,detailed mem \
3) statistics total use mem
=head1 Version
Author: Li linji, lilinji@genomics.cn
Version: 1.0, Date: 2012-7-30
=head1 Usage
--queue set you want look up queue (defined general.q)
--mem set Use full details queue Jobs and mem (check over_mem,and jods-ID)
--s set Task sort by (name,job,vf,mem && must set --mem)##defined name
--t set Task state information (defined run)
--help output help information to screen
=head1 Exmple
./sge_cluster_queue.pl
perl sge_cluster_queue.pl
perl sge_cluster_queue.pl -queue general.q
perl sge_cluster_queue.pl -queue general.q -mem
perl sge_cluster_queue.pl -queue general.q -mem -s vf
=cut
use Getopt::Long;
use FindBin qw($Bin $Script);
use File::Basename qw(basename dirname);
use Data::Dumper;
my $Sort||="name";
my $St||="r";
my ($help,$mem);
$queue_search ||= "general.q";
GetOptions(
"help"=>\$help,
"queue=s"=>\$queue_search,
"mem"=>\$mem,
"s=s"=>\$Sort,
"t=s"=>\$St,
);
die `pod2text $0` if ($Help);
if ($Sort eq "1") {
$Sort="name";
}elsif($Sort eq "2"){
$Sort="job";
}elsif($Sort eq "3"){
$Sort="vf";
}elsif($Sort eq "4"){
$Sort="mem";
}
if ($Sort ne "vf" && $Sort ne "mem" && $Sort ne "name" && $Sort ne "job" ) {
print STDERR <<SORt;
-s : 1 or name : sort by name (default)
2 or job : sort by jobs number
3 or vf : sort by vf (need -m )
4 or mem : sort by mem (need -m )
SORt
exit 0;
if (!defined $mem && ($Sort eq "mem" || $Sort eq "vf" )) {
print STDERR <<SORt;
-mem : get mem info
-s :
3 or vf : sort by vf (need -m )
4 or mem : sort by mem (need -m )
SORt
exit 0;
}
}
###############################################################################
#my $group=shift;
#my $queue_search = shift;
######### ......, . grp ... ....################################################
################################################################################
### ....look users;
my @User;
my @users;
my $users=();
########may be change here#####
my $user_start=`cat /etc/passwd |grep 'home'|cut -f 1 -d :`;
chomp ($user_start);
@User=split /\n/,$user_start;
chomp(@User);
for (my $i=0;$i<@User;$i++){
push @users,"$User[$i]";
}
chomp (@users);
###################print userlist;
printf "the (公司) information list<<<<<<<<<<<<<<<( . o . )\n";
printf "User: @users\n";
printf "Please Look Queue information:>>>>>>>>O(._.)O~\n";
my %temp_users =();
#########################
#Definition search"; #
#########################
my $listinfo=`qstat -u @users`; ##### Look all queue_test
my @task = split /\n/,$listinfo;
shift @task ;shift @task ;
my %out=();##$out{...}
my %statue=();## ..
my %Queue=();### ..
my %Job=();
my %For_sort=();### .sort ...sort ... $For_sort{$name}=info for sort;
my $jobs='';
################################### look each anyone test;
foreach my $job ( @task )
{
$job=~s/^\s+//;
my @tab=split /\s+/,$job;#### $tab[0] = job id ; $tab[3]= ... ; $tab[4] = .. ; $tab[7] = .. ;
if ($tab[7]=~/(\S+)\@(compute-\d+-\d+)\./)
{
my $queue=$1;
my $compute = $2;
if ($queue eq $queue_search)
{
#$queue = $compute_grp{$compute};
}
else
{
next;
}
$Queue{$queue}=1;
$Job{$tab[0]}{'queue'}=$queue;
if (exists $out{$tab[3]}{'queue'}{$queue})
{########## .... ....
$out{$tab[3]}{'queue'}{$queue}++;
}
else
{
$out{$tab[3]}{'queue'}{$queue}=1;
}
}
$statue{$tab[4]}=1;
$Job{$tab[0]}{'status'}=$tab[4];
$Job{$tab[0]}{'user'}=$tab[3];
###### ..sort
if ($Sort eq "name")
{
$For_sort{$tab[3]}=$tab[3];
}
elsif($Sort eq "job" && $tab[4] eq $St)
{
$For_sort{$tab[3]}++;
}
######
$jobs.=$tab[0].',';
if (exists $out{$tab[3]}{'status'}{$tab[4]} )
{ ########.. ..
$out{$tab[3]}{'status'}{$tab[4]}++;
}
else
{
$out{$tab[3]}{'status'}{$tab[4]}=1;
}
}
######################### .. ..group .... ...... ..... ....... .... > .... . ..id ..... - .....
my $overload=0;
if (defined $mem) {
my $detail=`qstat -j $jobs`;###... .. ....
my @jobinfos=split /={2,}/,$detail;####.......
foreach my $info (@jobinfos) {
if ($info =~/job_number:\s+(\d+)/) {
my $jobnum=$1;### job id
my $user=$Job{$jobnum}{'user'};
my $status=$Job{$jobnum}{'status'};
my $hard_mem=0;
if ($info =~/hard resource_list:\s+virtual_free=(\S+)\n/) {
$hard_mem=$1;### ... ..
$hard_mem= MEM_GMKT ($hard_mem);
### .. sort
if ($Sort eq "vf" && $St eq $status) {
$For_sort{$user}+=$hard_mem;
}
if (exists $out{$user}{'hard'}{'status'}{$status}) {
$out{$user}{'hard'}{'status'}{$status}+=$hard_mem;
}else{
$out{$user}{'hard'}{'status'}{$status}=$hard_mem;
}
if (exists $Job{$jobnum}{'queue'}) {
my $queue=$Job{$jobnum}{'queue'};
if (exists $out{$user}{'hard'}{'queue'}{$queue}) {
$out{$user}{'hard'}{'queue'}{$queue}+=$hard_mem;
}else{
$out{$user}{'hard'}{'queue'}{$queue}=$hard_mem;
}
my $vmem=0;
while ($info =~/vmem=(\S+), maxvmem=\S+/g) {
my $mmm=$1;
if ($mmm ne 'N/A') {
$vmem+=MEM_GMKT ($mmm);
}
}
### .. sort look over test list;
if ($Sort eq "mem" && $St eq $status) {
$For_sort{$user}+=$vmem;
}
my $over=$vmem-$hard_mem;
if ($over >0) {
my $intover=(int (10*$over+0.5))/10;
if (exists $out{$user}{'overid'}) {
$out{$user}{'overid'}.=','.$jobnum;
$out{$user}{'overmem'}.=','.$intover;
$out{$user}{'overnum'}+=1;
$out{$user}{'totalover'}+=$over;
}else{
$out{$user}{'overid'}=$jobnum;
$out{$user}{'overmem'}=$intover;
$out{$user}{'overnum'}=1;
$out{$user}{'totalover'}=$over;
}
$overload++;
}
if (exists $out{$user}{'mem'}{'queue'}{$queue}) {
$out{$user}{'mem'}{'queue'}{$queue}+=$vmem;
}else{
$out{$user}{'mem'}{'queue'}{$queue}=$vmem;
}
if (exists $out{$user}{'mem'}{'status'}{$status}) {
$out{$user}{'mem'}{'status'}{$status}+=$vmem;
}else{
$out{$user}{'mem'}{'status'}{$status}=$vmem;
}
}
}
}
}
}
########################################################
########### sort
################################
my @sort_name=sort keys %For_sort;
if ($Sort ne "name") {
foreach my $k (keys %out) {
if (!exists $For_sort{$k}) {
$For_sort{$k}=0;
}
}
@sort_name=sort {$For_sort{$a} <=> $For_sort{$b}} keys %For_sort;
}
########################################################
########### .. ..
################################
if (defined $mem) {
printf "\n";
printf "#################################################################################################################################\n";
printf "%11s",'user';
print "\t".'number of jobs , vf (G) , vmem (G) '."\n";
}
printf "%11s",'user';
my %total=();
my ($stat,$que,$user);
########### .. ..
foreach $stat (keys %statue) {
print ' ';
printf "%12s",$stat;
$total{'status'}{$stat}=0;
if (defined $mem) {
$total{'hard'}{'status'}{$stat}=0;
$total{'mem'}{'status'}{$stat}=0;
}
}
foreach $que (keys %Queue) {
print ' ';
printf "%12s",$queue_search;
$total{'queue'}{$que}=0;
if (defined $mem) {
$total{'hard'}{'queue'}{$que}=0;
$total{'mem'}{'queue'}{$que}=0;
}
}
if ($overload>0) {
print "\t"."over_jobs\tover_mem\t".'overload_id\'s'."\t".'overload_mem(G)';
}
print "\n";
######## group ... .. ..
foreach $user (@sort_name ) {
printf "%12s",$user;
foreach $stat (keys %statue) { ######## .. ..
my $outtab='';
if (exists $out{$user}{'status'}{$stat}) {
$outtab.=$out{$user}{'status'}{$stat};
$total{'status'}{$stat}+=$out{$user}{'status'}{$stat};
if (defined $mem) {
if (exists $out{$user}{'hard'}{'status'}{$stat}) {
my $hard=(int (10*$out{$user}{'hard'}{'status'}{$stat} + 0.5))/10;
$outtab.=','.$hard;
$total{'hard'}{'status'}{$stat}+=$out{$user}{'hard'}{'status'}{$stat};
}else{
$outtab.=',-';
}
if (exists $out{$user}{'mem'}{'status'}{$stat}) {
my $vmem=(int (10*$out{$user}{'mem'}{'status'}{$stat} + 0.5))/10;
$outtab.=','.$vmem;
$total{'mem'}{'status'}{$stat}+=$out{$user}{'mem'}{'status'}{$stat};
}else{
$outtab.=',-';
}
}
}else{
$outtab.='-';
if (defined $mem) {
$outtab.=',-,-';
}
}
print ' ';
printf "%12s",$outtab;
}
foreach $que (keys %Queue) { ########## .... .....
my $outtab='';
if (exists $out{$user}{'queue'}{$que} ) {
$outtab.=$out{$user}{'queue'}{$que};
$total{'queue'}{$que}+=$out{$user}{'queue'}{$que};
if (defined $mem) {
if (exists $out{$user}{'hard'}{'queue'}{$que}) {
my $hard=(int (10*$out{$user}{'hard'}{'queue'}{$que} + 0.5))/10;
$outtab.=','.$hard;
$total{'hard'}{'queue'}{$que}+=$out{$user}{'hard'}{'queue'}{$que};
}else{
$outtab.=',-';
}
if (exists $out{$user}{'mem'}{'queue'}{$que}) {
my $vmem=(int (10*$out{$user}{'mem'}{'queue'}{$que} + 0.5))/10;
$outtab.=','.$vmem;
$total{'mem'}{'queue'}{$que}+=$out{$user}{'mem'}{'queue'}{$que};
}else{
$outtab.=',-';
}
}
}else{
$outtab.='-';
if (defined $mem) {
$outtab.=',-,-';
}
}
print ' ';
printf "%12s",$outtab;
}
if ($overload>0) {
if (exists $out{$user}{'overid'} ) {
print "\t".$out{$user}{'overnum'}."\t".(int(10*$out{$user}{'totalover'}+0.5)/10)."\t".$out{$user}{'overid'}."\t".$out{$user}{'overmem'};;
}else{
print "\t".'---'."\t".'---'."\t".'---'."\t".'---';
}
}
print "\n";
}
printf "%11s",'total';
############.. total
foreach $stat (keys %statue) {
my $outtab='';
$outtab.=$total{'status'}{$stat};
if (defined $mem) {
my $hard=(int(10*$total{'hard'}{'status'}{$stat}+0.5))/10;
$outtab.=','.$hard;
my $vmem=(int(10*$total{'mem'}{'status'}{$stat}+0.5))/10;
$outtab.=','.$vmem;
}
print ' ';
printf "%12s",$outtab;
}
foreach $que (keys %Queue) {
my $outtab='';
$outtab.=$total{'queue'}{$que};
if (defined $mem) {
my $hard=(int(10*$total{'hard'}{'queue'}{$que}+0.5))/10;
$outtab.=','.$hard;
my $vmem=(int(10*$total{'mem'}{'queue'}{$que}+0.5))/10;
$outtab.=','.$vmem;
}
print ' ';
printf "%12s",$outtab;
}
print "\n";
printf "#################################################################################################################################\n";print "\n";
###########################################
############# ... .... .sub over test ID #
###########################################
sub MEM_GMKT{
my $num=shift;
if ($num=~/(\S+)g/i) {
$num=$1;
}elsif ($num=~/(\S+)m/i) {
$num=$1/1024;
}elsif ($num=~/(\S+)k/i) {
$num=$1/1048576;## 1048576 =1024*1024
}elsif ($num=~/(\S+)t/i) {
$num=$1*1024;##
}else{
}
return $num;
}
__END__
#################################################################################################################################
user number of jobs , vf (G) , vmem (G)
user r Eqw qw general.q over_jobs over_mem overload_id's overload_mem(G)
dengdj 5,15,21.9 -,-,- -,-,- 5,15,21.9 3 7.7 7120606,7120607,7120609 0.9,1,5.8
heh 35,105,22.1 -,-,- -,-,- 35,105,22.1 --- --- --- ---
huangl 1,5,0.3 -,-,- -,-,- 1,5,0.3 --- --- --- ---
jiangchb 49,98,19.3 -,-,- 1,2,- 49,98,19.3 --- --- --- ---
linfq -,-,- 3,6,- -,-,- -,-,- --- --- --- ---
sunhy 50,300,74.6 -,-,- -,-,- 50,300,74.6 --- --- --- ---
wangml 15,45,37.6 -,-,- -,-,- 15,45,37.6 7 10.1 7120577,7120586,7120594,7120708,7120869,7120898,7120907 1.6,1.6,0.8,1.7,1.1,1.6,1.6
zenghp -,-,- -,-,- 1,5,- -,-,- --- --- --- ---
zhangjie -,-,- 2,4,- -,-,- -,-,- --- --- --- ---
total 155,568,175.7 5,10,0 2,7,0 155,568,175.7
#################################################################################################################################
README
user: the group queue general.q owner
state(
r: "number of jobs , vf (G) , vmem (G)" 1:numbers job 2:vf= mem 3:Actual use
Eqw: 1:numbers job 2:vf= mem 3:Actual use
qw: 1:numbers job 2:vf= mem 3:Actual use
..
)
general.q: The current query queue
over_jobs: over_jobs.number
over_mem: over_mem.size
overload_id's: the job-ID
overload_mem(G) the job-ID's overload_mem
推荐阅读：
Linux 高可用（HA）集群之heartbeat基于crm进行资源管理详解