#!/usr/local/bin/ruby18
=begin

.$$$$       $.                                                .$$$$       $.                            
$$$$$       $$$$. .$$$$  $$$$$ .$$$$$$$.    .$$$$$$$$$$$$$$$. $$$$$       $$$$. .$$$$$$$$$. .$$$$$$$.    
$ $$$       $$$$$ $ $$$  $$$$$ $ $$$$$$$$$. $$$$$ $ $$$ $$$$$ $ $$$       $$$$$ $ $$$  $$$$ $ $$$$$$$$. 
$  $$       $$$$$ $  $$  $$$$$ $  $$  $$$$$ $$$$$ $  $$ $$$$$ $  $$       $$$$$ $  $$       $  $$  $$$$' 
$..$$       $$$$$ $..$$$$$$$$$ $..$$$$$$$$$ `:$:' $..$$ `:$:' $..$$       $$$$$ $..$$       $..$$$$$$$. 
$:::$       $$$$$ $:::$$$$$$$$ $:::$$$$$$$$       $:::$       $:::$       $$$$$ $:::$$$$    $:::$  $$$$$ 
$;;;$   $   $$$$$ $;;;$  $$$$$ $;;;$  $$$$$       $;;;$       $;;;$   $   $$$$$ $;;;$       $;;;$  $$$$$ 
$$$$$  $$$  $$$$$ $$$$$  $$$$$ $$$$$  $$$$$       $$$$$       $$$$$  $$$  $$$$$ $$$$$   ;$$ $$$$$  $$$$$ 
$$$$$$$$ $$$$$$$$ $$$$$  $$$$$ $$$$$  $$$$$       $$$$$       $$$$$$$$ $$$$$$$$ $$$$$$$$$$$ $$$$$$$$$$$' 


WhatWeb - Next generation web scanner.
Author: Andrew Horton aka urbanadventurer. Security Consultant for Security-Assessment.com
 
Homepage: http://www.morningstarsecurity.com/research/whatweb



Copyright 2009-2011 Andrew Horton <andrew.horton at security-assessment dot com>

This file is part of WhatWeb.

WhatWeb is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 2 of the License, or
(at your option) any later version.

WhatWeb is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
GNU General Public License for more details.

You should have received a copy of the GNU General Public License
along with WhatWeb.  If not, see <http://www.gnu.org/licenses/>.
=end

# encoding: utf-8
# coding: utf-8

#require 'profile' # for debugging
require 'getoptlong'
require 'pp'
require 'net/http'
require 'open-uri'
require 'cgi'
require 'thread'
require 'iconv'

# load resolv-replace to see whatweb on speed
begin
	require 'rubygems' # rubygems is optional
	if Gem.available?('em-resolv-replace') # this is much faster
		require 'resolv'
		require 'resolv-replace' # does this work with ruby 1.9?
	end
rescue LoadError
	# that failed.. no big deal
end

# load JSON if it's available. Is this Ruby 1.9 JSON safe? who knows?
begin
	require 'rubygems' # rubygems is optional
	if Gem.available?('json') # 
		require 'json'
	end
rescue LoadError
	# that failed.. no big deal
end

# load MongoDB if it's available. Is this Ruby 1.9 JSON safe? who knows?
begin
	require 'rubygems' # rubygems is optional
	if Gem.available?('mongo') # 
		require 'mongo'
	end
rescue LoadError
	# that failed.. no big deal
end

# load rchardet -- needed for foreign charsets and mongodb
begin
	require 'rubygems' # rubygems is optional
	if Gem.available?('rchardet') # this detects encodings and is cpu heavy
		require 'rchardet'
	end
rescue LoadError
	# that failed.. no big deal
end



if RUBY_VERSION =~ /^1\.9/
        require 'digest/md5'
else
        require 'md5'
	require 'net/https'
end


# add the directory of the file currently being executed to the load path
$LOAD_PATH.unshift(File.expand_path(File.dirname(__FILE__))) unless
    $:.include?(File.dirname(__FILE__)) || $LOAD_PATH.include?(File.expand_path(File.dirname(__FILE__)))
$LOAD_PATH << "/usr/local/share/whatweb/"

# if __FILE__ is a symlink then follow *every* symlink
if File.symlink?(__FILE__)
  require 'pathname'
  $LOAD_PATH << File.dirname( Pathname.new(__FILE__).realpath )
end

require 'lib/plugins.rb'
require 'lib/output.rb'
require 'lib/colour.rb'
require 'lib/tld.rb'
require 'lib/anemone/anemone.rb'
require 'tempfile'


#unless opts.size
# look through LOAD_PATH for the following plugin directories. Could be in same dir as whatweb or /usr/local/share/whatweb, etc
PLUGIN_DIRS=[ "plugins", "my-plugins"].map {|x| $LOAD_PATH.map {|y| y+"/"+x if File.exists?(y+"/"+x) } }.flatten.compact

$DEBUG = false # raise exceptions in plugins, etc
$VERSION="0.4.7"
$verbose=0
$USE_EXAMPLE_URLS=false
$use_colour="auto"
$USER_AGENT="WhatWeb/#{$VERSION}"
$MAX_THREADS=25
$AGGRESSION=1
$RECURSIVE=false
$RECURSIVE_DEPTH=10
$MAX_LINKS_TO_FOLLOW=250
$FOLLOW_REDIRECT="always"
$MAX_REDIRECTS=10
$USE_PROXY=false
$PROXY_HOST=nil
$PROXY_PORT=8080
$PROXY_USER=nil
$PROXY_PASS=nil
$URL_PREFIX=""
$URL_SUFFIX=""
$URL_PATTERN=nil
$NO_THREADS=false
$HTTP_OPEN_TIMEOUT=15
$HTTP_READ_TIMEOUT=30
$ANEMONE_SKIP_EXTENSIONS = %w|zip gz tar jpg exe png pdf|
$ANEMONE_SKIP_REGEX=nil
$WAIT=nil
$OUTPUT_ERRORS=nil
$QUIET=false
$CUSTOM_HEADERS={}
$BASIC_AUTH_USER=nil
$BASIC_AUTH_PASS=nil
$PLUGIN_TIMES=Hash.new(0)

# precompile regular expressions in plugins for performance
def precompile_regular_expressions
	Plugin.registered_plugins.each do |thisplugin|
		matches=thisplugin[1].matches
		unless matches.nil?
			matches.each do |thismatch|
				unless thismatch[:regexp].nil?
					#pp thismatch
					thismatch[:regexp_compiled]=Regexp.new(thismatch[:regexp])					
				end

				[:version, :os, :string, :account, :model, :firmware, :module, :filepath].each do |label|
					if !thismatch[label].nil? and thismatch[label].class==Regexp
						thismatch[:regexp_compiled]=Regexp.new(thismatch[label])
						#pp thismatch
					end
				end

				unless thismatch[:text].nil?
					thismatch[:regexp_compiled]=Regexp.new(Regexp.escape(thismatch[:text]))
				end
			end
		end		
	end
end

def plugin_list
	puts "WhatWeb Plugin List"
	puts
	puts ["Plugin Name".ljust(25),"Description"].join(" ")
	puts "-" * 79
	Plugin.registered_plugins.delete_if {|n,p| n == "\302\277" }.sort_by {|a,b| a.downcase }.each do |n,p|
		puts [n.ljust(25), (p.description.delete("\n\r") unless p.description.nil?)].join(" ")[0..78]
	end
	puts "-" * 30
	puts "#{Plugin.registered_plugins.size} Plugins Loaded"
	puts
end

# takes a string and returns an array of lines. used by plugin_info
def word_wrap(s,width=10)
	ret=[]
	line=""
	s.split.map {|x|
		word=x
		if line.size + x.size + 1 <= width
			line += x + " "
		else
			if word.size > width
				ret << line
				line = ""
				w=word.clone
				while w.size > width
					ret << w[0..(width-1)]
					w=w[width.to_i..-1]
				end
				ret << w unless w.size == 0
			else
				ret << line
				line=x + " "
			end
		end		
 	}
	ret << line
	ret
end


def plugin_info(keywords= nil)
	puts "WhatWeb Plugin Information"
	puts "Searching for " + keywords.join(",") unless keywords.nil?
	puts "-" * 80

	puts ["Plugin Name".ljust(25),"Details"].join(" ")
	count=0
	Plugin.registered_plugins.delete_if {|n,p| n == "\302\277" }.sort_by {|a,b| a.downcase }.each do |name,plugin|
		# not include examples	
		dump=[name,plugin.author,plugin.description,plugin.matches].flatten.compact.to_a.join.downcase
		if keywords.empty? or keywords.map {|k| dump.include?(k.downcase) }.compact.include?(true)
			puts name
			puts "\tAuthor:".ljust(22) + plugin.author
			puts "\tVersion:".ljust(22) +plugin.version
#			puts "\tCategory:".ljust(22) +plugin.category.to_s unless plugin.category.nil?
			puts "\tExamples:".ljust(22) +plugin.examples.size.to_s if plugin.examples
			puts "\tMatches:".ljust(22) +plugin.matches.size.to_s if plugin.matches
			puts "\tPassive function: ".ljust(22) + (defined?(plugin.passive) ? "Yes" : "No")
			puts "\tAggressive function: ".ljust(22) + (defined?(plugin.aggressive) ? "Yes" : "No")
			puts "\tVersion detection: ".ljust(22) + (plugin.version_detection? ? "Yes" : "No")
		
			if plugin.description
				puts "\tDescription: "
				word_wrap(plugin.description,72).each {|line| puts "\t" + line }
			end

			puts
			puts "-" * 80
			count+=1
		end
	end
	puts "#{count} plugins found"
	puts
end







# This is used in plugin selection by load_plugins
class PluginChoice
	attr_accessor :modifier, :type, :name

	def <=>(s)
		x = -1 if self.modifier.nil?
		x = 0 if self.modifier=="+"
		x = 1 if self.modifier=="-"
		x
	end

	def fill(s)
		self.modifier = nil
		self.modifier=s[0].chr if ["+","-"].include?(s[0].chr)

		if self.modifier
			self.name=s[1..-1]
		else
			self.name=s
		end

		# figure out and store the filename or pluginname
		if File.exists?(File.expand_path(self.name))
			self.type = "file"
			self.name = File.expand_path(self.name)
		else
			self.name.downcase!
			self.type = "plugin"
		end
	end
end

# this is used by load_plugins
def load_plugin(f)
	begin
		load f
	rescue
		error("Error: failed to load #{f}")
	rescue SyntaxError
		error("Error: Failed to load #{f}.")
	end
end


=begin
for adding/removing sets of plugins.

--plugins +plugins-disabled,-foobar (+ adds to the full set, -removes from the fullset. items can be directories, files or plugin names)
--plugins +/tmp/moo.rb
--plugins foobar (only select foobar)
--plugins ./plugins-disabled,-md5 (select only plugins from the plugins-disabled folder, remove the md5 plugin from the selected list)
=end
def load_plugins(list=nil)
	# separate l into a and b
	#	a = make list of dir & filenames
	#	b = make list of assumed pluginnames
	a=[];b=[]

	plugin_dirs=PLUGIN_DIRS.clone

	plugin_dirs.map {|p| p=File.expand_path(p) }

	if list
		list=list.split(",")

		plugins_disabled_location = ["plugins-disabled"].map {|x| $LOAD_PATH.map {|y| y+"/"+x if File.exists?(y+"/"+x) } }.flatten.compact

		list.each {|x| x.gsub!(/^\+$/,"+#{plugins_disabled_location}") } # + is short for +plugins-disabled

		list.each do |p|
			choice=PluginChoice.new
			choice.fill(p)
			a << choice if choice.type == "file"
			b << choice if choice.type == "plugin"
		end

		# sort by neither, add, minus
		a=a.sort

		if a.map {|c| c.modifier }.include?(nil)
			plugin_dirs=[]
		end

		minus_files = [] # make list of files not to load
		a.map {|c|
			plugin_dirs << c.name if c.modifier.nil? or c.modifier == "+"
			plugin_dirs -= [c.name] if c.modifier == "-" # for Dirs
			minus_files << c.name if c.modifier == "-"    # for files
		}
		
		# load files from plugin_dirs unless a file is minused
		plugin_dirs.each do |d|
			# if a folder, then load all files
			if File.directory?(d)
				(Dir.glob("#{d}/*.rb")-minus_files).each {|x| load_plugin(x) }
			elsif File.exists?(d)
				load_plugin(d)
			else
				error("Error: #{d} is not Dir or File")
			end
		end
		
		# make list of plugins to run
		# go through all plugins, remove from list any that match b minus
		selected_plugin_names=[]

		if b.map {|c| c.modifier }.include?(nil)
			selected_plugin_names=[]
		else
			selected_plugin_names = Plugin.registered_plugins.map {|n,p| n.downcase }
		end

		b.map {|c| 
			selected_plugin_names << c.name if c.modifier.nil? or c.modifier=="+"
			selected_plugin_names -= [c.name] if c.modifier == "-"
		}
		plugins_to_use = Plugin.registered_plugins.map {|n,p| [n,p] if selected_plugin_names.include?(n.downcase) }.compact

		# report on plugins that couldn't be found
		unfound_plugins = selected_plugin_names - plugins_to_use.map {|n,p| n.downcase }
		unless unfound_plugins.empty?
			puts "Error: The following plugins were not found: " + unfound_plugins.join(",")
		end

	else
		# no selection, so it's default
		plugin_dirs.each do |d|
			Dir.glob("#{d}/*.rb").each {|x|
				load_plugin(x)
			}
		end
		plugins_to_use = Plugin.registered_plugins
	end

	plugins_to_use
end






def custom_plugin(c)
	# define a custom plugin on the cmdline
	# ":text=>'powered by abc'" or
	# "{:text=>'powered by abc'},{:regexp=>/abc [ ]?1/i}"

	# then it's ok..
	if c =~ /:(text|ghdb|md5|regexp|tagpattern)=>[\/'"].*/
		matches="matches [\{#{c}\}]"
	end

	# this isn't checked for sanity... loading plugins = cmd exec anyway
	if c =~ /\{.*\}/
		matches="matches [#{c}]"
	end

	abort("Invalid custom plugin syntax: #{c}") if matches.nil?

	custom="Plugin.define \"Custom-Plugin\" do
	author \"Unknown\"
	description \"User defined\"
	#{matches}
	end
	"

	begin
		# open tmp file
		f=Tempfile.new('whatweb-custom-plugin')
		# write
		f.write(custom)
		f.close
		# load
		load f.path
		f.unlink
		true
	rescue SyntaxError
		error("Error: Cannot load custom plugin")
		false
	end
end


def make_target_list(cmdline_args, inputfile=nil,pluginlist = nil)
	url_list = cmdline_args

	# read each line as a url, skipping lines that begin with a #
	if !inputfile.nil? and File.exists?(inputfile)
		pp "loading input file: #{inputfile}" if $verbose > 2
		url_list += File.read(inputfile).to_a.each {|line| line.strip! }.delete_if {|line| line =~ /^#.*/ }.each {|line| line.delete!("\n") }
	end

	# add example urls to url_list if required. plugins must be loaded already
	if $USE_EXAMPLE_URLS
		url_list += pluginlist.map {|name,plugin| plugin.examples unless plugin.examples.nil? }.compact.flatten
	end

        genrange=url_list.map do |x|
	  range=nil
	  if x =~ /^[0-9\.\-*\/]+$/ and not x =~ /^[\d\.]+$/
	    # check for nmap
	    error "Target ranges require nmap to be in the path" if `which nmap` == ""
	    range=`nmap -n -sL #{x} 2>&1 | egrep -o "([0-9]{1,3}\\.){3}[0-9]{1,3}"`
	    range=range.split("\n")
	  end
	  range
	end.compact.flatten
	
	url_list= url_list.select {|x| not x =~ /^[0-9\.\-*\/]+$/ or x =~ /^[\d\.]+$/ }
	url_list += genrange unless genrange.empty?
     

	#make urls friendlier, test if it's a file, if test for not assume it's http://
	# http, https, ftp, etc
	url_list=url_list.map do |x|   
		if File.exists?(x)
			x
		else
			# use url pattern
			if $URL_PATTERN
				x = $URL_PATTERN.gsub('%insert%',x)
			end
			# add prefix & suffix
			x=$URL_PREFIX + x + $URL_SUFFIX

			if x =~ (/^[a-z]+:\/\//)
				x
			else
				x.sub(/^/,"http://")
			end
		end	
	end

	url_list=url_list.flatten #.sort.uniq
end


def certainty_to_words(p)
	case p
		when 0..49
			"maybe"
		when 50..99
			"probably"
		when 100
			"certain"
	end
end

def match_ghdb(ghdb, body, meta, status, base_uri)
	# this could be made faster by creating code to eval once for each plugin

	pp "match_ghdb",ghdb if $verbose > 2
	
	# take a GHDB string and turn it into code to be evaluated
	matches=[] # fill with true or false. succeeds if all true
	s = ghdb

	# does it contain intitle?
	if s =~ /intitle:/i
		# extract either the next word or the following words enclosed in "s, it can't possibly be both
		intitle = (s.scan(/intitle:"([^"]*)"/i) + s.scan(/intitle:([^"]\w+)/i)).to_s
		matches << ((body =~ /<title>[^<]*#{intitle}[^<]*<\/title>/i).nil? ? false : true)
		# strip out the intitle: part
		s=s.gsub(/intitle:"([^"]*)"/i,'').gsub(/intitle:([^"]\w+)/i,'')
	end

	if s =~ /filetype:/i
		filetype = (s.scan(/filetype:"([^"]*)"/i) + s.scan(/filetype:([^"]\w+)/i)).to_s
		# lame method: check if the URL ends in the filetype
		unless base_uri.nil?
			matches << ((base_uri.path.split("?")[0] =~ /#{filetype}$/i).nil? ? false : true)
		end
		s=s.gsub(/filetype:"([^"]*)"/i,'').gsub(/filetype:([^"]\w+)/i,'')
	end

	if s =~ /inurl:/i
		inurl = (s.scan(/inurl:"([^"]*)"/i) + s.scan(/inurl:([^"]\w+)/i)).flatten	
		# can occur multiple times.
		inurl.each {|x| matches << ((base_uri.to_s =~ /#{inurl}/i).nil? ? false : true)  }
		# strip out the filetype: part
		s=s.gsub(/inurl:"([^"]*)"/i,'').gsub(/inurl:([^"]\w+)/i,'')
	end

	# split the remaining words except those enclosed in quotes, remove the quotes and sort them

	remaining_words = s.scan(/([^ "]+)|("[^"]+")/i).flatten.compact.each {|w| w.delete!('"')  }.sort.uniq
	
	pp "Remaining GHDB words", 	remaining_words if $verbose > 2
	
	remaining_words.each do |w| 	
		# does it start with a - ?
		if w[0..0] == '-'
			# reverse true/false if it begins with a -
			matches << ((body =~ /#{w[1..-1]}/i).nil? ? true : false) 
		else
			w = w[1..-1] if w[0..0] == '+' # if it starts with +, ignore the 1st char
			matches << ((body =~ /#{w}/i).nil? ? false : true)
		end	
	end

	pp matches if $verbose > 2

	# if all matches are true, then true	
	if matches.uniq == [true]
		true
	else
		false
	end
end


def error(s)
	if defined?($semaphore)
		# We want the output mutex locked.
		# Has our current thread already locked the Mutex?
		begin
			$semaphore.lock
		rescue ThreadError
			# we're already locked. Nice.		
		end
	end
	if ($use_colour=="auto") or ($use_colour=="always")
		STDERR.puts red(s)
	else
		STDERR.puts s
	end
	STDERR.flush
	unless $OUTPUT_ERRORS.nil?
		$OUTPUT_ERRORS.out(s)
	end
	$semaphore.unlock if defined?($semaphore)
end


def open_target(target)
	# follow 301's
	begin
		uri=URI.parse(target)
		path=uri.path
		path="/" if uri.path==""
		query=uri.query
		
		if $USE_PROXY == true
			http=Net::HTTP::Proxy($PROXY_HOST,$PROXY_PORT, $PROXY_USER, $PROXY_PASS).new(uri.host,uri.port)
		else
			http=Net::HTTP.new(uri.host,uri.port)
		end
		
		# set timeouts
		http.open_timeout = $HTTP_OPEN_TIMEOUT
		http.read_timeout = $HTTP_READ_TIMEOUT

		#puts path -- path doesn't include parameters
		
		# if it's https://
		# i wont worry about certificates, verfication, etc
		if uri.class == URI::HTTPS
			http.use_ssl = true	
			http.verify_mode = OpenSSL::SSL::VERIFY_NONE		
		end
		
		

		req=Net::HTTP::Get.new(path + (query.nil? ? "" : "?" + query ) , $CUSTOM_HEADERS)

		if $BASIC_AUTH_USER	
			req.basic_auth $BASIC_AUTH_USER, $BASIC_AUTH_PASS
		end
		res=http.request(req)
		
		headers={}; res.each_header {|x,y| headers[x]=y }
		body=res.body
		status=res.code.to_i
		puts uri.host.to_s + path + (query.nil? ? "" : "?" + query ) + " [#{status}]" if  $verbose > 0 
		cookies=res.get_fields('set-cookie')
		
	rescue SocketError => err
		error(target + " ERROR: Socket error #{err}")
		return [0, nil, nil, nil,nil]
	rescue TimeoutError => err
		error(target + " ERROR: Timed out #{err}")
		return [0, nil, nil, nil,nil]
	rescue Errno::ETIMEDOUT	=>err # for ruby 1.8.7 patch level 249
		error(target + " ERROR: Timed out (ETIMEDOUT) #{err}")
		return [0, nil, nil, nil,nil]
	rescue EOFError => err
		error(target + " ERROR: EOF error #{err}")
		return [0, nil, nil, nil,nil]
	rescue StandardError => err		
		err = "Not HTTP or cannot resolve hostname" if err.to_s == "undefined method `closed?' for nil:NilClass"
		error(target + " ERROR: #{err}")
		return [0, nil, nil, nil,nil]
	rescue => err
		error(target + " ERROR: #{err}")
		return [0, nil, nil, nil,nil]
	end


	begin		
		ip=IPSocket.getaddress(uri.host)
	rescue StandardError => err		
		err = "Cannot resolve hostname" if err.to_s == "undefined method `closed?' for nil:NilClass"
		error(target + " ERROR: #{err}")
		return [0, nil, nil, nil,nil]
	end
	[status,uri,ip,body,headers,cookies]
end


def run_plugins(target,body,headers=nil,cookies=nil,status=nil,url=nil,ip="")
		#pp target, body, headers, status, url

		# need a webpage model for this stuff
		if body.nil?
			md5sum=nil
			tagpattern=nil
			# Initialize @body variable if the connection is terminated prematurely
			# This is usually caused by HTTP status codes: 101, 102, 204, 205, 305
			body=""
		else
			md5sum=Digest::MD5.hexdigest(body) 		
			tagpattern = make_tag_pattern(body)
		end
		#########################################

		# is the target a file or URL?
		target_is_file=false
		target_is_url=true if target =~ /^http[s]?:\/\//

		results=[]
		$plugins_to_use.each do |name,plugin|
			begin			
				while plugin.locked?
					sleep 0.1
					puts "Waiting for plugin:#{name} to unlock" if $verbose > 2
				end
				plugin.lock

				if target_is_url
					#plugin.init(ObjectSpace._id2ref(body.object_id),headers,cookies,status,url,ip,md5sum,tagpattern)
					plugin.init(body,headers,cookies,status,url,ip,md5sum,tagpattern)
				else
					# it's a file
					plugin.init(body,{},nil,0,URI.parse('http://example.com/' + target),"",md5sum,tagpattern)
				end
				
				# eXecute the plugin			
				#start_time = Time.now
				result=plugin.x
				#end_time = Time.now
				#$PLUGIN_TIMES[name] += end_time - start_time

				plugin.unlock

			rescue StandardError => err
				error("ERROR: Plugin #{name} failed for #{target.to_s}. #{err}")
				plugin.unlock
				raise if $DEBUG==true
			end
			results << [name, result] unless result.nil? or result.empty?
		end
	results
end

def make_tag_pattern(b)
	# remove stuff between script and /script
	# don't bother with  !--, --> or noscript and /noscript
	inscript=false;

	b.scan(/<([^\s>]*)/).flatten.map {|x| x.downcase!; r=nil;
			r=x if inscript==false
			inscript=true if x=="script"
			(inscript=false; r=x) if x=="/script"
			r
		}.compact.join(",")
end


def usage()
puts"
.$$$     $.                                   .$$$     $.         
$$$$     $$. .$$$  $$$ .$$$$$$.  .$$$$$$$$$$. $$$$     $$. .$$$$$$$. .$$$$$$. 
$ $$     $$$ $ $$  $$$ $ $$$$$$. $$$$$ $$$$$$ $ $$     $$$ $ $$   $$ $ $$$$$$.
$ `$     $$$ $ `$  $$$ $ `$  $$$ $$' $ `$ `$$ $ `$     $$$ $ `$      $ `$  $$$'
$. $     $$$ $. $$$$$$ $. $$$$$$ `$  $. $  :' $. $     $$$ $. $$$$   $. $$$$$.
$::$  .  $$$ $::$  $$$ $::$  $$$     $::$     $::$  .  $$$ $::$      $::$  $$$$
$;;$ $$$ $$$ $;;$  $$$ $;;$  $$$     $;;$     $;;$ $$$ $$$ $;;$      $;;$  $$$$
$$$$$$ $$$$$ $$$$  $$$ $$$$  $$$     $$$$     $$$$$$ $$$$$ $$$$$$$$$ $$$$$$$$$'

"
puts "WhatWeb - Next generation web scanner.\nVersion #{$VERSION} by Andrew Horton aka urbanadventurer from Security-Assessment.com"
puts "Homepage: http://www.morningstarsecurity.com/research/whatweb"
puts
puts "Usage: whatweb [options] <URLs>"
puts "
TARGET SELECTION:
  <URLs>\t\tEnter URLs, filenames or nmap-format IP ranges.
\t\t\tUse /dev/stdin to pipe HTML directly
  --input-file=FILE, -i\tIdentify URLs found in FILE, eg. -i /dev/stdin
  --url-prefix\t\tAdd a prefix to target URLs
  --url-suffix\t\tAdd a suffix to target URLs
  --url-pattern\t\tInsert the targets into a URL. Requires --input-file,
\t\t\teg. www.example.com/%insert%/robots.txt 
  --example-urls, -e\tAdd example URLs for each selected plugin to the target
\t\t\tlist. By default will add example URLs for all plugins.

AGGRESSION LEVELS:
  --aggression, -a=LEVEL The aggression level controls the trade-off between
\t\t\tspeed/stealth and reliability. Default: 1
\t\t\tAggression levels are:
\t1 (Passive)\tMake one HTTP request per target. Except for redirects.
\t2 (Polite)\tReserved for future use
\t3 (Aggressive)\tTriggers aggressive plugin functions only when a
\t\t\tplugin matches passively.
\t4 (Heavy)\tTrigger aggressive functions for all plugins. Guess a
\t\t\tlot of URLs like Nikto.

HTTP OPTIONS:
  --user-agent, -U=AGENT Identify as AGENT instead of WhatWeb/#{$VERSION}.
  --user, -u=<user:password> HTTP basic authentication
  --header, -H\t\tAdd an HTTP header. eg \"Foo:Bar\". Specifying a default
\t\t\theader will replace it. Specifying an empty value, eg.
\t\t\t\"User-Agent:\" will remove the header.
  --follow-redirect=WHEN Control when to follow redirects. WHEN may be `never',
\t\t\t`http-only', `meta-only', `same-site', `same-domain'
\t\t\tor `always'. Default: #{$FOLLOW_REDIRECT}
  --max-redirects=NUM\tMaximum number of contiguous redirects. Default: 10

SPIDERING:
  --recursion, -r\tFollow links recursively. Only follow links under the
\t\t\tpath Default: off
  --depth, -d\t\tMaximum recursion depth. Default: #{$RECURSIVE_DEPTH}
  --max-links, -m\tMaximum number of links to follow on one page
\t\t\tDefault: #{$MAX_LINKS_TO_FOLLOW}
  --spider-skip-extensions Redefine extensions to skip.
\t\t\tDefault: #{$ANEMONE_SKIP_EXTENSIONS.join(",")}

PROXY:
  --proxy\t\t<hostname[:port]> Set proxy hostname and port
\t\t\tDefault: #{$PROXY_PORT}
  --proxy-user\t\t<username:password> Set proxy user and password

PLUGINS:
  --plugins, -p\t\tComma delimited set of selected plugins. Default is all.
\t\t\tEach element can be a directory, file or plugin name and
\t\t\tcan optionally have a modifier, eg. + or -
\t\t\tExamples: +/tmp/moo.rb,+/tmp/foo.rb
\t\t\ttitle,md5,+./plugins-disabled/
\t\t\t./plugins-disabled,-md5
\t\t\t-p + is a shortcut for -p +plugins-disabled
  --list-plugins, -l\tList the plugins
  --info-plugins, -I\tDisplay information for all plugins. Optionally search
\t\t\twith keywords in a comma delimited list.
  --custom-plugin\tDefine a custom plugin called Custom-Plugin,
\t\t\tExamples: \":text=>'powered by abc'\"
\t\t\t\":regexp=>/powered[ ]?by ab[0-9]/\"
\t\t\t\":ghdb=>'intitle:abc \\\"powered by abc\\\"'\"
\t\t\t\":md5=>'8666257030b94d3bdb46e05945f60b42'\"
\t\t\t\"{:text=>'powered by abc'},{:regexp=>/abc [ ]?1/i}\"

LOGGING & OUTPUT:
  --verbose, -v\t\tIncrease verbosity, use twice for plugin development.
  --colour,--color=WHEN\tcontrol whether colour is used. WHEN may be `never',
\t\t\t`always', or `auto'
  --quiet, -q\t\tDo not display brief logging to STDOUT
  --log-brief=FILE\tLog brief, one-line output
  --log-verbose=FILE\tLog verbose output
  --log-xml=FILE\tLog XML format
  --log-json=FILE\tLog JSON format
  --log-json-verbose=FILE Log JSON Verbose format
  --log-magictree=FILE\tLog MagicTree XML format
  --log-object=FILE\tLog Ruby object inspection format
  --log-mongo-database\tName of the MongoDB database
  --log-mongo-collection Name of the MongoDB collection. Default: whatweb
  --log-mongo-host\tMongoDB hostname or IP address. Default: 0.0.0.0
  --log-mongo-username\tMongoDB username. Default: nil
  --log-mongo-password\tMongoDB password. Default: nil
  --log-errors=FILE\tLog errors

PERFORMANCE & STABILITY:
  --max-threads, -t\tNumber of simultaneous threads. Default: #{$MAX_THREADS}.
  --open-timeout\tTime in seconds. Default: #{$HTTP_OPEN_TIMEOUT}
  --read-timeout\tTime in seconds. Default: #{$HTTP_READ_TIMEOUT}
  --wait=SECONDS\tWait SECONDS between connections
\t\t\tThis is useful when using a single thread.

HELP & MISCELLANEOUS:
  --help, -h\t\tThis help
  --debug\t\tRaise errors in plugins
  --version\t\tDisplay version information. (WhatWeb #{$VERSION})

EXAMPLE USAGE:
  whatweb example.com
  whatweb -v example.com
  whatweb -a 3 example.com
  whatweb 192.168.1.0/24
\n"

	suggestions=""
	suggestions << "WARNING: Without the em-resolve-replace gem performance is significantly degraded.\n" unless Gem.available?('em-resolv-replace')
	suggestions << "To enable JSON logging install the json gem.\n" unless Gem.available?('json')
	suggestions << "To enable MongoDB logging install the mongo gem.\n" unless Gem.available?('mongo')
	suggestions << "To enable character set detection and MongoDB logging install the rchardet gem.\n" unless Gem.available?('rchardet')

	unless suggestions.empty?
		print "\nOPTIONAL DEPENDENCIES\n--------------------------------------------------------------------------------\n" + suggestions + "\n"
	end
end


if ARGV.size==0 # faster usage info
	usage
	exit 
end

plugin_selection=nil
use_custom_plugin=false
input_file=nil
output_list = []
mongo={}
mongo[:use_mongo_log]=false



opts = GetoptLong.new(
      [ '--help', '-h', GetoptLong::NO_ARGUMENT ],
      [ '-v','--verbose', GetoptLong::NO_ARGUMENT ],
      [ '-l','--list-plugins', GetoptLong::NO_ARGUMENT ],
      [ '-p','--plugins', GetoptLong::REQUIRED_ARGUMENT ],
      [ '-I','--info-plugins', GetoptLong::OPTIONAL_ARGUMENT ],
      [ '-e','--example-urls', GetoptLong::NO_ARGUMENT ],
      [ '--colour','--color', GetoptLong::REQUIRED_ARGUMENT ],
      [ '--log-object', GetoptLong::REQUIRED_ARGUMENT ],
      [ '--log-brief', GetoptLong::REQUIRED_ARGUMENT ],
      [ '--log-xml', GetoptLong::REQUIRED_ARGUMENT ],
      [ '--log-json', GetoptLong::REQUIRED_ARGUMENT ],
      [ '--log-json-verbose', GetoptLong::REQUIRED_ARGUMENT ],
      [ '--log-magictree', GetoptLong::REQUIRED_ARGUMENT ],
      [ '--log-verbose', GetoptLong::REQUIRED_ARGUMENT ],
      [ '--log-mongo-collection', GetoptLong::REQUIRED_ARGUMENT ],
      [ '--log-mongo-host', GetoptLong::REQUIRED_ARGUMENT ],
      [ '--log-mongo-database', GetoptLong::REQUIRED_ARGUMENT ],
      [ '--log-mongo-username', GetoptLong::REQUIRED_ARGUMENT ],
      [ '--log-mongo-password', GetoptLong::REQUIRED_ARGUMENT ],
      [ '--log-errors', GetoptLong::REQUIRED_ARGUMENT ],
      [ '-i','--input-file', GetoptLong::REQUIRED_ARGUMENT ],
      [ '-U','--user-agent', GetoptLong::REQUIRED_ARGUMENT ],
      [ '-a','--aggression', GetoptLong::REQUIRED_ARGUMENT ],
      [ '-t','--max-threads', GetoptLong::REQUIRED_ARGUMENT ],
      [ '-m','--max-links', GetoptLong::REQUIRED_ARGUMENT ],
      [ '-r','--recursive', GetoptLong::NO_ARGUMENT ],
      [ '-d','--depth', GetoptLong::REQUIRED_ARGUMENT ],
      [ '--follow-redirect', GetoptLong::REQUIRED_ARGUMENT ],
      [ '--max-redirects', GetoptLong::REQUIRED_ARGUMENT ],
      [ '--proxy', GetoptLong::REQUIRED_ARGUMENT ],
      [ '--proxy-user', GetoptLong::REQUIRED_ARGUMENT ],
      [ '--url-prefix', GetoptLong::REQUIRED_ARGUMENT ],
      [ '--url-suffix', GetoptLong::REQUIRED_ARGUMENT ],
      [ '--url-pattern', GetoptLong::REQUIRED_ARGUMENT ],
      [ '--custom-plugin', GetoptLong::REQUIRED_ARGUMENT ],
      [ '--open-timeout', GetoptLong::REQUIRED_ARGUMENT ],
      [ '--read-timeout', GetoptLong::REQUIRED_ARGUMENT ],
      [ '--spider-skip-extensions', GetoptLong::REQUIRED_ARGUMENT ],
      [ '--header','-H', GetoptLong::REQUIRED_ARGUMENT ],
      [ '--user','-u', GetoptLong::REQUIRED_ARGUMENT ],
      [ '--wait', GetoptLong::REQUIRED_ARGUMENT ],
      [ '--debug', GetoptLong::NO_ARGUMENT ],
      [ '--version', GetoptLong::NO_ARGUMENT ],
      [ '-q','--quiet', GetoptLong::NO_ARGUMENT]
    )

begin    
	opts.each do |opt, arg|
		case opt
			when '-i','--input-file'
				input_file=arg
			when '-l','--list-plugins'
				load_plugins
				plugin_list
				exit
			when '-p','--run-plugins'
				plugin_selection=arg
			when '-I','--info-plugins'                
				load_plugins
				plugin_info(arg.split(","))
				exit
			when '-e','--example-urls'
				$USE_EXAMPLE_URLS=true
			when '--color','--colour'
				case arg
					when 'auto'
						$use_colour="auto"
					when 'always'
						$use_colour="always"
					when 'never'
						$use_colour=false
					else
						raise("--colour argument not recognized")
					end
			when '--log-object'
				output_list << OutputObject.new(arg)
			when '--log-brief'
			 	output_list << OutputBrief.new(arg)
			when '--log-xml'
			 	output_list << OutputXML.new(arg)
			when '--log-magictree'
				output_list << OutputMagicTreeXML.new(arg)
			when '--log-verbose'
				output_list << OutputVerbose.new(arg)
			when '--log-json'
				if defined?(JSON)
			 		output_list << OutputJSON.new(arg)
				else
					raise("Sorry. The JSON gem is required for JSON output")
				end
			when '--log-json-verbose'
				if defined?(JSON)
			 		output_list << OutputJSONVerbose.new(arg)
				else
					raise("Sorry. The JSON gem is required for JSONVerbose output")
				end
			when '--log-mongo-collection'
				if defined?(Mongo) and defined?(CharDet)
					mongo[:collection]=arg
					mongo[:use_mongo_log]=true
				else
					raise("Sorry. The mongo and rchardet gems are required for Mongo output")
				end

			when '--log-mongo-host'
				if defined?(Mongo) and defined?(CharDet)
			 		mongo[:host]=arg
					mongo[:use_mongo_log]=true
				else
					raise("Sorry. The mongo and rchardet gems are required for Mongo output")
				end

			when '--log-mongo-database'
				if defined?(Mongo) and defined?(CharDet)
			 		mongo[:database]=arg
					mongo[:use_mongo_log]=true
				else
					raise("Sorry. The mongo and rchardet gems are required for Mongo output")
				end
			when '--log-mongo-username'
				if defined?(Mongo) and defined?(CharDet)
			 		mongo[:username]=arg
					mongo[:use_mongo_log]=true
				else
					raise("Sorry. The mongo and rchardet gems are required for Mongo output")
				end
			when '--log-mongo-password'
				if defined?(Mongo) and defined?(CharDet)
			 		mongo[:password]=arg
					mongo[:use_mongo_log]=true
				else
					raise("Sorry. The mongo and rchardet gems are required for Mongo output")
				end
			when '--log-errors'
			 	$OUTPUT_ERRORS = OutputErrors.new(arg)
			when '-U','--user-agent'
				$USER_AGENT=arg
			when '-t','--max-threads'
				$MAX_THREADS=arg.to_i
			when '-a','--aggression'
				$AGGRESSION=arg.to_i
			when '-r','--recursive'
				$RECURSIVE=true
			when '-d','--depth'
				$RECURSIVE_DEPTH=arg.to_i
			when '-m','--max-links'	    		
				$MAX_LINKS_TO_FOLLOW=arg.to_i
			when '--proxy'
				$USE_PROXY=true
				$PROXY_HOST = arg.to_s.split(":")[0]
				$PROXY_PORT = arg.to_s.split(":")[1].to_i if arg.to_s.include?(":")		
			when '--proxy-user'
				$PROXY_USER=arg.to_s.split(":")[0]
				$PROXY_PASS=arg.to_s.scan(/^[^:]+:(.+)/).to_s if arg =~ /^[^:]+:(.+)/
			when '-q','--quiet'
				$QUIET=true
			when '--url-prefix'
				$URL_PREFIX=arg
			when '--url-suffix'
				$URL_SUFFIX=arg
			when '--url-pattern'
				$URL_PATTERN=arg
			when '--custom-plugin'				
				use_custom_plugin=true if custom_plugin(arg)
			when '--follow-redirect'	
				if ["never","http-only","meta-only","same-site","same-domain","always"].include?(arg.downcase)
					$FOLLOW_REDIRECT=arg.downcase
				else
					raise("Invalid --follow-redirect parameter.")
				end
			when '--max-redirects'
				$MAX_REDIRECTS=arg.to_i
			when '--open-timeout'					
				$HTTP_OPEN_TIMEOUT=arg.to_i
			when '--read-timeout'					
				$HTTP_READ_TIMEOUT=arg.to_i
			when '--spider-skip-extensions'
				$ANEMONE_SKIP_EXTENSIONS = arg.split(",")
			when '--wait'
				$WAIT = arg.to_i
			when '-H','--header'
				begin
					x=arg.scan(/([^:]+):(.*)/).flatten
					raise if x.empty?
					$CUSTOM_HEADERS[x.first]=x.last
				rescue
					raise("Invalid --header parameter.")
				end
			when '-u','--user'
				$BASIC_AUTH_USER=arg.split(":").first
				$BASIC_AUTH_PASS=arg.to_s.scan(/^[^:]+:(.+)/).to_s if arg =~ /^[^:]+:(.+)/
			when '--debug'
				$DEBUG = true
			when '-h','--help'
				usage
				exit
			when '-v','--verbose'
				$verbose=$verbose+1
			when '--version'
				puts "WhatWeb version #{$VERSION} ( http://www.morningstarsecurity.com/research/whatweb/ )"
				exit
		end
	end		
rescue StandardError, GetoptLong::Error => err
	puts
	usage
	puts err
	exit
end

# sanity checks
if $AGGRESSION >= 3 and $RECURSIVE
	error("Aggressive levels 3 or higher are not compatible with recursive spidering.")
	exit 1
end

### PLUGINS
plugin_selection += ",Custom-Plugin" if use_custom_plugin and plugin_selection  
$plugins_to_use = load_plugins(plugin_selection ) # load all the plugins

# are the no plugins?
if $plugins_to_use.size == 0
	error "No plugins selected, exiting."
	exit 1
end
precompile_regular_expressions # optimise plugins


### OUTPUT
output_list << OutputBrief.new unless $QUIET # by default output brief
output_list << OutputObject.new() if $verbose > 1 # full output if -vv
output_list <<  OutputVerbose.new() if $verbose > 0 # full output if -v

if mongo[:use_mongo_log]
	if $plugins_to_use.map { |a,b| a }.include?("Charset")
		output_list << OutputMongo.new(mongo) 
	else
		error("MongoDB logging requires the Charset plugin to be activated. It is not included by default for speed considerations.")
		exit
	end
end

## Headers
$CUSTOM_HEADERS["User-Agent"]=$USER_AGENT unless $CUSTOM_HEADERS["User-Agent"]
$CUSTOM_HEADERS.delete_if {|k,v| v=="" }


### TARGETS
# clean up urls, add example urls if needed
$targets=make_target_list(ARGV, input_file, $plugins_to_use)
$recent_targets=[]
pp "Targets: " ,$targets if $verbose>1

# fail & show usage if no targets.
if $targets.size <1
	error "No targets selected"	
	usage
	exit 1
end


# set up ANEMONE skip regex
# turn an array of extensions into a regexp
$ANEMONE_SKIP_REGEX=Regexp.union($ANEMONE_SKIP_EXTENSIONS.map{|x| "/\.#{" + x + "}$/"  }.to_s())

# for each target, we try each plugin then print the results
$semaphore=Mutex.new
Thread.abort_on_exception = true if $DEBUG
meta_refresh_regex=/<meta[\s]+http\-equiv[\s]*=[\s]*['"]?refresh['"]?[^>]+content[\s]*=[^>]*[0-9]+;[\s]*url=['"]?([^"^'^>]+)['"]?[^>]*>/i


def next_target
	t=nil

	#pp $targets

	# this semaphore may not be required
	$semaphore.synchronize do
		while !Thread.main["targets"].empty?
			$targets.push Thread.main["targets"].pop
		end
	end

	while $recent_targets.include?(t) or t.nil?
		t=$targets.pop
		#puts "t is ...#{t}" 

		# t at the end of the $targets list
		if t.nil?
			puts "t is nil" if $verbose > 2
			if $targets.empty?
				#puts "yes, targets are emtpy"
				if Thread.main["targets"].size >0
					new=Thread.main["targets"].pop
					$targets << new
				elsif Thread.list.size > 1
					puts "Thread list size: #{Thread.list.size}" if $verbose > 2
					sleep 1
				else
					puts "breaking now" if $verbose > 2
					break
				end
			end
		end
	end

	puts "Recent Targets:" + $recent_targets.join(",") if $verbose > 2

	$recent_targets.push t
	$recent_targets.pop if $recent_targets.size > 100 # we dont need to care about mroe than 100

	#puts "next target: #{t}"

	t
end

Thread.main["targets"]=[]

while thistarget = next_target
		Thread.new do
			# if u swap around the following lines the target gets confused. idk why.
			target = thistarget # we set the target within the thread
			puts Thread.current.to_s + " started for " + target if $verbose>1

			sleep $WAIT unless $WAIT.nil? # wait
			# get the webpage	
			# get the file/webpage, and return statuscode, base url, html body, html headers
			# if target is a file
			target_is_a_file=File.exists?(target)
		
			if $RECURSIVE == true and target_is_a_file == false

				begin		
					ip=IPSocket.getaddress(URI::parse(target).host)
				rescue StandardError => err		
					err = "Cannot resolve hostname" if err.to_s == "undefined method `closed?' for nil:NilClass"
					ip=""
				end

				Anemone.crawl(target,
				{"threads"=>1, "user_agent"=>$USER_AGENT, "depth_limit"=>$RECURSIVE_DEPTH}) do |anemone|
				anemone.skip_links_like $ANEMONE_SKIP_REGEX
						anemone.focus_crawl { |page| page.links.slice(0..$MAX_LINKS_TO_FOLLOW) }
						anemone.on_every_page do |page|
							sleep $WAIT unless $WAIT.nil? # wait if required
							# convert headers
							headers=Hash.new
							unless page.headers.nil?
								page.headers.each {|k,v| headers[k]=v.to_s }
							end

							#pp target,page.doc,page.headers,page.code,page.url				
							unless page.original_doc.nil?
								doc=page.original_doc
								status=page.code
								url=page.url
# no @cookies for anemone :(
								results = run_plugins(url.to_s,doc,headers,nil,status,url,ip)
								# reporting
								# multiple output plugins simultaneously, some stdout, some files

								output_list.each do |o|
									begin
										o.out(url, status, results)
									rescue
										#srsly, logging failed
										error("Error: Logging failed for "+ target)
									end
								end

							end

						end
				end
		
			else
			# Recursive is false
		
			# follow redirects
			no_redirects =false
			num_redirects = 0
			while no_redirects == false do
				# if we redirect 10 times we give up
				if num_redirects == $MAX_REDIRECTS
					error(target + " ERROR: Too many redirects")
					no_redirects=true
					next
				end
		
				# either not recursive, or a file
				if target_is_a_file == true
					# target is a file
					doc=open(target)
					body=doc.read
					no_redirects=true
				else
					# not a file, not recursive
					#status,url,body,headers = open_with_openuri(target)
					status,url,ip,body,headers,cookies = open_target(target)
					if status == 0 or status.nil?
						no_redirects=true
						next
					end
				end
				#results = run_plugins(target,ObjectSpace._id2ref(body.object_id),headers,cookies,status,url,ip)
				results = run_plugins(target,body,headers,cookies,status,url,ip)
			
				# reporting
				# multiple output plugins simultaneously, some stdout, some files

				output_list.each do |o|
					begin
						o.out(target, status, results)
					rescue
						#srsly, logging failed
						error("Error: Logging failed for "+ target)
					end
				end

				# REDIRECTION
				begin
					newtarget_m=nil
					newtarget_h=nil
					newtarget=nil			

					if meta_refresh_regex =~ body
						metarefresh=body.scan(meta_refresh_regex).first.to_s
						newtarget_m=URI.join(target,metarefresh).to_s # this works for relative and absolute
					end

					unless status.nil? or headers.nil?
						newtarget_h=URI.join(target,headers["location"]).to_s if (300..399) === status and headers["location"]
					end
					# if both meta refresh location and HTTP location are set, then the HTTP location overrides

					if newtarget_m or newtarget_h
						case $FOLLOW_REDIRECT
						when "never"
							no_redirects=true
						when "http-only"
							newtarget = newtarget_h
						when "meta-only"
							newtarget = newtarget_m
						when "same-site"
							newtarget = (newtarget_h or newtarget_m) if URI.parse((newtarget_h or newtarget_m)).host == URI.parse(target).host # defaults to _h if both are present
						when "same-domain"
							newtarget = (newtarget_h or newtarget_m) if TLD.same_domain?(
								URI.parse(target).host, URI.parse((newtarget_h or newtarget_m)).host)
						when "always"
							newtarget = (newtarget_h or newtarget_m)
						else
							error("Error: Invalid REDIRECT mode")
						end
					end

					# if we found a new target to following, then set it and increment our redirect counter
					if newtarget
						unless newtarget == target
							num_redirects+=1 
							target=newtarget
						else
							no_redirects=true
						end
					else
						# no redirects is set to true because we've failed to set a redirect
						no_redirects=true 
					end
				rescue => err
					error("Redirection broken: #{err}")
					no_redirects=true
				end

			end # while no_redirects
			end # if $RECURSIVE
	
			end # Thread.new

			while Thread.list.size>($MAX_THREADS+1)
				puts "sleeping" if $verbose>1
				sleep 0.5
			end
end # targets.each


# close output logs
output_list.each {|o| o.close }

# shutdown plugins
Plugin.registered_plugins.map {|name,plugin| plugin.shutdown }

#pp $PLUGIN_TIMES.sort_by {|x,y|y }

