Subversion Repositories DevTools

#!/usr/bin/env perl

# cloc -- Count Lines of Code                  {{{1

# Copyright (C) 2006-2015 Al Danial <al.danial@gmail.com>

# First release August 2006

#

# Includes code from:

#   - SLOCCount v2.26 

#     http://www.dwheeler.com/sloccount/

#     by David Wheeler.

#   - Regexp::Common v2.120

#     http://search.cpan.org/~abigail/Regexp-Common-2.120/lib/Regexp/Common.pm

#     by Damian Conway and Abigail.

#   - Win32::Autoglob 

#     http://search.cpan.org/~sburke/Win32-Autoglob-1.01/Autoglob.pm

#     by Sean M. Burke.

#   - Algorithm::Diff

#     http://search.cpan.org/~tyemq/Algorithm-Diff-1.1902/lib/Algorithm/Diff.pm

#     by Tye McQueen.

#

# This program is free software; you can redistribute it and/or modify

# it under the terms of the GNU General Public License as published by

# the Free Software Foundation; either version 2 of the License, or

# (at your option) any later version.

#

# This program is distributed in the hope that it will be useful,

# but WITHOUT ANY WARRANTY; without even the implied warranty of

# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the

# GNU General Public License for more details:

# <http://www.gnu.org/licenses/gpl.txt>.

#

# 1}}}

my $VERSION = "1.65";  # odd number == beta; even number == stable

my $URL     = "https://github.com/AlDanial/cloc";

require 5.006;

# use modules                                  {{{1

use warnings;

use strict;

use Getopt::Long;

use File::Basename;

use File::Temp qw { tempfile tempdir };

use File::Find;

use File::Path;

use File::Spec;

use IO::File;

use POSIX "strftime";

# Digest::MD5 isn't in the standard distribution. Use it only if installed.

my $HAVE_Digest_MD5 = 0;

eval "use Digest::MD5;";

if (defined $Digest::MD5::VERSION) {

    $HAVE_Digest_MD5 = 1;

} else {

    warn "Digest::MD5 not installed; will skip file uniqueness checks.\n";

}

# Time::HiRes became standard with Perl 5.8

my $HAVE_Time_HiRes = 0;

eval "use Time::HiRes;";

$HAVE_Time_HiRes = 1 if defined $Time::HiRes::VERSION;

my $HAVE_Rexexp_Common;

# Regexp::Common isn't in the standard distribution.  It will

# be installed in a temp directory if necessary.

BEGIN {

    if (eval "use Regexp::Common;") {

        $HAVE_Rexexp_Common = 1;

    } else {

        $HAVE_Rexexp_Common = 0;

    }

}

my $HAVE_Algorith_Diff = 0;

# Algorithm::Diff isn't in the standard distribution.  It will

# be installed in a temp directory if necessary.

eval "use Algorithm::Diff qw ( sdiff ) ";

if (defined $Algorithm::Diff::VERSION) {

    $HAVE_Algorith_Diff = 1;

} else {

    Install_Algorithm_Diff();

}

# print "2 HAVE_Algorith_Diff = $HAVE_Algorith_Diff\n";

# test_alg_diff($ARGV[$#ARGV - 1], $ARGV[$#ARGV]); die;

# Uncomment next two lines when building Windows executable with perl2exe

# or if running on a system that already has Regexp::Common.

#use Regexp::Common;

#$HAVE_Rexexp_Common = 1;

#perl2exe_include "Regexp/Common/whitespace.pm"

#perl2exe_include "Regexp/Common/URI.pm"

#perl2exe_include "Regexp/Common/URI/fax.pm"

#perl2exe_include "Regexp/Common/URI/file.pm"

#perl2exe_include "Regexp/Common/URI/ftp.pm"

#perl2exe_include "Regexp/Common/URI/gopher.pm"

#perl2exe_include "Regexp/Common/URI/http.pm"

#perl2exe_include "Regexp/Common/URI/pop.pm"

#perl2exe_include "Regexp/Common/URI/prospero.pm"

#perl2exe_include "Regexp/Common/URI/news.pm"

#perl2exe_include "Regexp/Common/URI/tel.pm"

#perl2exe_include "Regexp/Common/URI/telnet.pm"

#perl2exe_include "Regexp/Common/URI/tv.pm"

#perl2exe_include "Regexp/Common/URI/wais.pm"

#perl2exe_include "Regexp/Common/CC.pm"

#perl2exe_include "Regexp/Common/SEN.pm"

#perl2exe_include "Regexp/Common/number.pm"

#perl2exe_include "Regexp/Common/delimited.pm"

#perl2exe_include "Regexp/Common/profanity.pm"

#perl2exe_include "Regexp/Common/net.pm"

#perl2exe_include "Regexp/Common/zip.pm"

#perl2exe_include "Regexp/Common/comment.pm"

#perl2exe_include "Regexp/Common/balanced.pm"

#perl2exe_include "Regexp/Common/lingua.pm"

#perl2exe_include "Regexp/Common/list.pm"

#perl2exe_include "File/Glob.pm"

use Text::Tabs qw { expand };

use Cwd qw { cwd };

use File::Glob;

# 1}}}

# Usage information, options processing.       {{{1

my $ON_WINDOWS = 0;

   $ON_WINDOWS = 1 if ($^O =~ /^MSWin/) or ($^O eq "Windows_NT");

if ($ON_WINDOWS and $ENV{'SHELL'}) {

    if ($ENV{'SHELL'} =~ m{^/}) {

        $ON_WINDOWS = 0;  # make Cygwin look like Unix

    } else {

        $ON_WINDOWS = 1;  # MKS defines $SHELL but still acts like Windows

    }

}

my $NN     = chr(27) . "[0m";  # normal

   $NN     = "" if $ON_WINDOWS or !(-t STDERR); # -t STDERR:  is it a terminal?

my $BB     = chr(27) . "[1m";  # bold

   $BB     = "" if $ON_WINDOWS or !(-t STDERR);

my $script = basename $0;

my $usage  = "

Usage: $script [options] <file(s)/dir(s)> | <set 1> <set 2> | <report files>

 Count, or compute differences of, physical lines of source code in the

 given files (may be archives such as compressed tarballs or zip files)

 and/or recursively below the given directories.

 ${BB}Input Options${NN}

   --extract-with=<cmd>      This option is only needed if cloc is unable

                             to figure out how to extract the contents of

                             the input file(s) by itself.

                             Use <cmd> to extract binary archive files (e.g.:

                             .tar.gz, .zip, .Z).  Use the literal '>FILE<' as

                             a stand-in for the actual file(s) to be

                             extracted.  For example, to count lines of code

                             in the input files

                                gcc-4.2.tar.gz  perl-5.8.8.tar.gz

                             on Unix use

                               --extract-with='gzip -dc >FILE< | tar xf -'

                             or, if you have GNU tar,

                               --extract-with='tar zxf >FILE<'

                             and on Windows use, for example:

                               --extract-with=\"\\\"c:\\Program Files\\WinZip\\WinZip32.exe\\\" -e -o >FILE< .\"

                             (if WinZip is installed there).

   --list-file=<file>        Take the list of file and/or directory names to

                             process from <file>, which has one file/directory

                             name per line.  Only exact matches are counted;

                             relative path names will be resolved starting from 

                             the directory where cloc is invoked.  

                             See also --exclude-list-file.

   --unicode                 Check binary files to see if they contain Unicode

                             expanded ASCII text.  This causes performance to

                             drop noticably.

 ${BB}Processing Options${NN}

   --autoconf                Count .in files (as processed by GNU autoconf) of

                             recognized languages.

   --by-file                 Report results for every source file encountered.

   --by-file-by-lang         Report results for every source file encountered

                             in addition to reporting by language.

   --count-and-diff <set1> <set2>    

                             First perform direct code counts of source file(s)

                             of <set1> and <set2> separately, then perform a diff 

                             of these.  Inputs may be pairs of files, directories, 

                             or archives.  See also --diff, --diff-alignment,

                             --diff-timeout, --ignore-case, --ignore-whitespace.

   --diff <set1> <set2>      Compute differences in code and comments between

                             source file(s) of <set1> and <set2>.  The inputs

                             may be pairs of files, directories, or archives.

                             Use --diff-alignment to generate a list showing

                             which file pairs where compared.  See also

                             --count-and-diff, --diff-alignment, --diff-timeout, 

                             --ignore-case, --ignore-whitespace.

   --diff-timeout <N>        Ignore files which take more than <N> seconds

                             to process.  Default is 10 seconds.

                             (Large files with many repeated lines can cause 

                             Algorithm::Diff::sdiff() to take hours.)

   --follow-links            [Unix only] Follow symbolic links to directories

                             (sym links to files are always followed).

   --force-lang=<lang>[,<ext>]

                             Process all files that have a <ext> extension

                             with the counter for language <lang>.  For

                             example, to count all .f files with the

                             Fortran 90 counter (which expects files to

                             end with .f90) instead of the default Fortran 77

                             counter, use

                               --force-lang=\"Fortran 90\",f

                             If <ext> is omitted, every file will be counted

                             with the <lang> counter.  This option can be

                             specified multiple times (but that is only

                             useful when <ext> is given each time).

                             See also --script-lang, --lang-no-ext.

   --force-lang-def=<file>   Load language processing filters from <file>,

                             then use these filters instead of the built-in

                             filters.  Note:  languages which map to the same 

                             file extension (for example:

                             MATLAB/Objective C/MUMPS/Mercury;  Pascal/PHP; 

                             Lisp/OpenCL; Lisp/Julia; Perl/Prolog) will be 

                             ignored as these require additional processing 

                             that is not expressed in language definition 

                             files.  Use --read-lang-def to define new 

                             language filters without replacing built-in 

                             filters (see also --write-lang-def).

   --ignore-whitespace       Ignore horizontal white space when comparing files

                             with --diff.  See also --ignore-case.

   --ignore-case             Ignore changes in case; consider upper- and lower-

                             case letters equivalent when comparing files with

                             --diff.  See also --ignore-whitespace.

   --lang-no-ext=<lang>      Count files without extensions using the <lang>

                             counter.  This option overrides internal logic

                             for files without extensions (where such files

                             are checked against known scripting languages

                             by examining the first line for #!).  See also

                             --force-lang, --script-lang.

   --max-file-size=<MB>      Skip files larger than <MB> megabytes when

                             traversing directories.  By default, <MB>=100.

                             cloc's memory requirement is roughly twenty times 

                             larger than the largest file so running with 

                             files larger than 100 MB on a computer with less 

                             than 2 GB of memory will cause problems.  

                             Note:  this check does not apply to files 

                             explicitly passed as command line arguments.

   --read-binary-files       Process binary files in addition to text files.

                             This is usually a bad idea and should only be

                             attempted with text files that have embedded

                             binary data.

   --read-lang-def=<file>    Load new language processing filters from <file>

                             and merge them with those already known to cloc.  

                             If <file> defines a language cloc already knows 

                             about, cloc's definition will take precedence.  

                             Use --force-lang-def to over-ride cloc's 

                             definitions (see also --write-lang-def ).

   --script-lang=<lang>,<s>  Process all files that invoke <s> as a #!

                             scripting language with the counter for language

                             <lang>.  For example, files that begin with

                                #!/usr/local/bin/perl5.8.8

                             will be counted with the Perl counter by using

                                --script-lang=Perl,perl5.8.8

                             The language name is case insensitive but the

                             name of the script language executable, <s>,

                             must have the right case.  This option can be

                             specified multiple times.  See also --force-lang,

                             --lang-no-ext.

   --sdir=<dir>              Use <dir> as the scratch directory instead of

                             letting File::Temp chose the location.  Files

                             written to this location are not removed at

                             the end of the run (as they are with File::Temp).

   --skip-uniqueness         Skip the file uniqueness check.  This will give

                             a performance boost at the expense of counting

                             files with identical contents multiple times

                             (if such duplicates exist).

   --stdin-name=<file>       Give a file name to use to determine the language

                             for standard input.

   --strip-comments=<ext>    For each file processed, write to the current

                             directory a version of the file which has blank

                             lines and comments removed.  The name of each

                             stripped file is the original file name with

                             .<ext> appended to it.  It is written to the

                             current directory unless --original-dir is on.

   --original-dir            [Only effective in combination with

                             --strip-comments]  Write the stripped files

                             to the same directory as the original files.

   --sum-reports             Input arguments are report files previously

                             created with the --report-file option.  Makes

                             a cumulative set of results containing the

                             sum of data from the individual report files.

   --unix                    Override the operating system autodetection

                             logic and run in UNIX mode.  See also

                             --windows, --show-os.

   --windows                 Override the operating system autodetection

                             logic and run in Microsoft Windows mode.

                             See also --unix, --show-os.

 ${BB}Filter Options${NN}

   --exclude-dir=<D1>[,D2,]  Exclude the given comma separated directories

                             D1, D2, D3, et cetera, from being scanned.  For

                             example  --exclude-dir=.cache,test  will skip

                             all files that have /.cache/ or /test/ as part

                             of their path.

                             Directories named .bzr, .cvs, .hg, .git, and

                             .svn are always excluded.

   --exclude-ext=<ext1>[,<ext2>[...]]

                             Do not count files having the given file name

                             extensions.

   --exclude-lang=<L1>[,L2,] Exclude the given comma separated languages

                             L1, L2, L3, et cetera, from being counted.

   --exclude-list-file=<file>  Ignore files and/or directories whose names

                             appear in <file>.  <file> should have one file

                             name per line.  Only exact matches are ignored;

                             relative path names will be resolved starting from 

                             the directory where cloc is invoked.  

                             See also --list-file.

   --include-lang=<L1>[,L2,] Count only the given comma separated languages

                             L1, L2, L3, et cetera.

   --match-d=<regex>         Only count files in directories matching the Perl

                             regex.  For example

                               --match-d='/(src|include)/'

                             only counts files in directories containing

                             /src/ or /include/.

   --not-match-d=<regex>     Count all files except those in directories

                             matching the Perl regex.

   --match-f=<regex>         Only count files whose basenames match the Perl

                             regex.  For example

                               --match-f='^[Ww]idget'

                             only counts files that start with Widget or widget.

   --not-match-f=<regex>     Count all files except those whose basenames

                             match the Perl regex.

   --skip-archive=<regex>    Ignore files that end with the given Perl regular

                             expression.  For example, if given

                               --skip-archive='(zip|tar(\.(gz|Z|bz2|xz|7z))?)'

                             the code will skip files that end with .zip,

                             .tar, .tar.gz, .tar.Z, .tar.bz2, .tar.xz, and

                             .tar.7z.

   --skip-win-hidden         On Windows, ignore hidden files.

 ${BB}Debug Options${NN}

   --categorized=<file>      Save names of categorized files to <file>.

   --counted=<file>          Save names of processed source files to <file>.

   --explain=<lang>          Print the filters used to remove comments for

                             language <lang> and exit.  In some cases the 

                             filters refer to Perl subroutines rather than

                             regular expressions.  An examination of the

                             source code may be needed for further explanation.

   --diff-alignment=<file>   Write to <file> a list of files and file pairs

                             showing which files were added, removed, and/or

                             compared during a run with --diff.  This switch

                             forces the --diff mode on.

   --help                    Print this usage information and exit.

   --found=<file>            Save names of every file found to <file>.

   --ignored=<file>          Save names of ignored files and the reason they

                             were ignored to <file>.

   --print-filter-stages     Print processed source code before and after 

                             each filter is applied.

   --show-ext[=<ext>]        Print information about all known (or just the

                             given) file extensions and exit.

   --show-lang[=<lang>]      Print information about all known (or just the

                             given) languages and exit.

   --show-os                 Print the value of the operating system mode

                             and exit.  See also --unix, --windows.

   -v[=<n>]                  Verbose switch (optional numeric value).

   --version                 Print the version of this program and exit.

   --write-lang-def=<file>   Writes to <file> the language processing filters

                             then exits.  Useful as a first step to creating

                             custom language definitions (see also

                             --force-lang-def, --read-lang-def).

 ${BB}Output Options${NN}

   --3                       Print third-generation language output.

                             (This option can cause report summation to fail

                             if some reports were produced with this option

                             while others were produced without it.)

   --by-percent  X           Instead of comment and blank line counts, show 

                             these values as percentages based on the value 

                             of X in the denominator:

                                X = 'c'   -> # lines of code

                                X = 'cm'  -> # lines of code + comments

                                X = 'cb'  -> # lines of code + blanks

                                X = 'cmb' -> # lines of code + comments + blanks

                             For example, if using method 'c' and your code

                             has twice as many lines of comments as lines 

                             of code, the value in the comment column will 

                             be 200%.  The code column remains a line count.

   --csv                     Write the results as comma separated values.

   --csv-delimiter=<C>       Use the character <C> as the delimiter for comma

                             separated files instead of ,.  This switch forces

   --out=<file>              Synonym for --report-file=<file>.

                             --csv to be on.

   --progress-rate=<n>       Show progress update after every <n> files are

                             processed (default <n>=100).  Set <n> to 0 to

                             suppress progress output (useful when redirecting

                             output to STDOUT).

   --quiet                   Suppress all information messages except for

                             the final report.

   --report-file=<file>      Write the results to <file> instead of STDOUT.

   --sql=<file>              Write results as SQL create and insert statements

                             which can be read by a database program such as

                             SQLite.  If <file> is -, output is sent to STDOUT.

   --sql-append              Append SQL insert statements to the file specified

                             by --sql and do not generate table creation

                             statements.  Only valid with the --sql option.

   --sql-project=<name>      Use <name> as the project identifier for the

                             current run.  Only valid with the --sql option.

   --sql-style=<style>       Write SQL statements in the given style instead

                             of the default SQLite format.  Currently, the 

                             only style option is Oracle.

   --sum-one                 For plain text reports, show the SUM: output line

                             even if only one input file is processed.

   --xml                     Write the results in XML.

   --xsl=<file>              Reference <file> as an XSL stylesheet within

                             the XML output.  If <file> is 1 (numeric one),

                             writes a default stylesheet, cloc.xsl (or

                             cloc-diff.xsl if --diff is also given).

                             This switch forces --xml on.

   --yaml                    Write the results in YAML.

";

#  Help information for options not yet implemented:

#  --inline                  Process comments that appear at the end

#                            of lines containing code.

#  --html                    Create HTML files of each input file showing

#                            comment and code lines in different colors.

$| = 1;  # flush STDOUT

my $start_time = get_time();

my (

    $opt_categorized          ,

    $opt_found                ,

    @opt_force_lang           ,

    $opt_lang_no_ext          ,

    @opt_script_lang          ,

    $opt_count_diff           ,

    $opt_diff                 ,

    $opt_diff_alignment       ,

    $opt_diff_timeout         ,

    $opt_html                 ,

    $opt_ignored              ,

    $opt_counted              ,

    $opt_show_ext             ,

    $opt_show_lang            ,

    $opt_progress_rate        ,

    $opt_print_filter_stages  ,

    $opt_v                    ,

    $opt_version              ,

    $opt_exclude_lang         ,

    $opt_exclude_list_file    ,

    $opt_exclude_dir          ,

    $opt_explain              ,

    $opt_include_lang         ,

    $opt_force_lang_def       ,

    $opt_read_lang_def        ,

    $opt_write_lang_def       ,

    $opt_strip_comments       ,

    $opt_original_dir         ,

    $opt_quiet                ,

    $opt_report_file          ,

    $opt_sdir                 ,

    $opt_sum_reports          ,

    $opt_unicode              ,

    $opt_no3                  ,   # accept it but don't use it

    $opt_3                    ,

    $opt_extract_with         ,

    $opt_by_file              ,

    $opt_by_file_by_lang      ,

    $opt_by_percent           ,

    $opt_xml                  ,

    $opt_xsl                  ,

    $opt_yaml                 ,

    $opt_csv                  ,

    $opt_csv_delimiter        ,

    $opt_match_f              ,

    $opt_not_match_f          ,

    $opt_match_d              ,

    $opt_not_match_d          ,

    $opt_skip_uniqueness      ,

    $opt_list_file            ,

    $opt_help                 ,

    $opt_skip_win_hidden      ,

    $opt_read_binary_files    ,

    $opt_sql                  ,

    $opt_sql_append           ,

    $opt_sql_project          ,

    $opt_sql_style            ,

    $opt_inline               ,

    $opt_exclude_ext          ,

    $opt_ignore_whitespace    ,

    $opt_ignore_case          ,

    $opt_follow_links         ,

    $opt_autoconf             ,

    $opt_sum_one              ,

    $opt_stdin_name           ,

    $opt_force_on_windows     ,

    $opt_force_on_unix        ,   # actually forces !$ON_WINDOWS

    $opt_show_os              ,

    $opt_skip_archive         ,

    $opt_max_file_size        ,   # in MB

   );

my $getopt_success = GetOptions(

   "by_file|by-file"                         => \$opt_by_file             ,

   "by_file_by_lang|by-file-by-lang"         => \$opt_by_file_by_lang     ,

   "categorized=s"                           => \$opt_categorized         ,

   "counted=s"                               => \$opt_counted             ,

   "include_lang|include-lang=s"             => \$opt_include_lang        ,

   "exclude_lang|exclude-lang=s"             => \$opt_exclude_lang        ,

   "exclude_dir|exclude-dir=s"               => \$opt_exclude_dir         ,

   "exclude_list_file|exclude-list-file=s"   => \$opt_exclude_list_file   ,

   "explain=s"                               => \$opt_explain             ,

   "extract_with|extract-with=s"             => \$opt_extract_with        ,

   "found=s"                                 => \$opt_found               ,

   "count_and_diff|count-and-diff"           => \$opt_count_diff          ,

   "diff"                                    => \$opt_diff                ,

   "diff-alignment|diff_alignment=s"         => \$opt_diff_alignment      ,

   "diff-timeout|diff_timeout=i"             => \$opt_diff_timeout        ,

   "html"                                    => \$opt_html                ,

   "ignored=s"                               => \$opt_ignored             ,

   "quiet"                                   => \$opt_quiet               ,

   "force_lang_def|force-lang-def=s"         => \$opt_force_lang_def      ,

   "read_lang_def|read-lang-def=s"           => \$opt_read_lang_def       ,

   "show_ext|show-ext:s"                     => \$opt_show_ext            ,

   "show_lang|show-lang:s"                   => \$opt_show_lang           ,

   "progress_rate|progress-rate=i"           => \$opt_progress_rate       ,

   "print_filter_stages|print-filter-stages" => \$opt_print_filter_stages ,

   "report_file|report-file=s"               => \$opt_report_file         ,

   "out=s"                                   => \$opt_report_file         ,

   "script_lang|script-lang=s"               => \@opt_script_lang         ,

   "sdir=s"                                  => \$opt_sdir                ,

   "skip_uniqueness|skip-uniqueness"         => \$opt_skip_uniqueness     ,

   "strip_comments|strip-comments=s"         => \$opt_strip_comments      ,

   "original_dir|original-dir"               => \$opt_original_dir        ,

   "sum_reports|sum-reports"                 => \$opt_sum_reports         ,

   "unicode"                                 => \$opt_unicode             ,

   "no3"                                     => \$opt_no3                 ,  # ignored

   "3"                                       => \$opt_3                   ,

   "v:i"                                     => \$opt_v                   ,

   "version"                                 => \$opt_version             ,

   "write_lang_def|write-lang-def=s"         => \$opt_write_lang_def      ,

   "xml"                                     => \$opt_xml                 ,

   "xsl=s"                                   => \$opt_xsl                 ,

   "force_lang|force-lang=s"                 => \@opt_force_lang          ,

   "lang_no_ext|lang-no-ext=s"               => \$opt_lang_no_ext         ,

   "yaml"                                    => \$opt_yaml                ,

   "csv"                                     => \$opt_csv                 ,

   "csv_delimeter|csv-delimiter=s"           => \$opt_csv_delimiter       ,

   "match_f|match-f=s"                       => \$opt_match_f             ,

   "not_match_f|not-match-f=s"               => \$opt_not_match_f         ,

   "match_d|match-d=s"                       => \$opt_match_d             ,

   "not_match_d|not-match-d=s"               => \$opt_not_match_d         ,

   "list_file|list-file=s"                   => \$opt_list_file           ,

   "help"                                    => \$opt_help                ,

   "skip_win_hidden|skip-win-hidden"         => \$opt_skip_win_hidden     ,

   "read_binary_files|read-binary-files"     => \$opt_read_binary_files   ,

   "sql=s"                                   => \$opt_sql                 ,

   "sql_project|sql-project=s"               => \$opt_sql_project         ,

   "sql_append|sql-append"                   => \$opt_sql_append          ,

   "sql_style|sql-style=s"                   => \$opt_sql_style           ,

   "inline"                                  => \$opt_inline              ,

   "exclude_ext|exclude-ext=s"               => \$opt_exclude_ext         ,

   "ignore_whitespace|ignore-whitespace"     => \$opt_ignore_whitespace   ,

   "ignore_case|ignore-case"                 => \$opt_ignore_case         ,

   "follow_links|follow-links"               => \$opt_follow_links        ,

   "autoconf"                                => \$opt_autoconf            ,

   "sum_one|sum-one"                         => \$opt_sum_one             ,

   "by_percent|by-percent=s"                 => \$opt_by_percent          ,

   "stdin_name|stdin-name=s"                 => \$opt_stdin_name          ,

   "windows"                                 => \$opt_force_on_windows    ,

   "unix"                                    => \$opt_force_on_unix       ,

   "show_os|show-os"                         => \$opt_show_os             ,

   "skip_archive|skip-archive=s"             => \$opt_skip_archive        ,

   "max_file_size|max-file-size=i"           => \$opt_max_file_size       ,

  );

$opt_by_file  = 1 if defined  $opt_by_file_by_lang;

my $CLOC_XSL = "cloc.xsl"; # created with --xsl

   $CLOC_XSL = "cloc-diff.xsl" if $opt_diff;

die "\n" unless $getopt_success;

die $usage if $opt_help;

my %Exclude_Language = ();

   %Exclude_Language = map { $_ => 1 } split(/,/, $opt_exclude_lang) 

        if $opt_exclude_lang;

my %Exclude_Dir      = ();

   %Exclude_Dir      = map { $_ => 1 } split(/,/, $opt_exclude_dir ) 

        if $opt_exclude_dir ;

my %Include_Language = ();

   %Include_Language = map { $_ => 1 } split(/,/, $opt_include_lang) 

        if $opt_include_lang;

# Forcibly exclude .svn, .cvs, .hg, .git, .bzr directories.  The contents of these

# directories often conflict with files of interest.

$opt_exclude_dir       = 1;

$Exclude_Dir{".svn"}   = 1;

$Exclude_Dir{".cvs"}   = 1;

$Exclude_Dir{".hg"}    = 1;

$Exclude_Dir{".git"}   = 1;

$Exclude_Dir{".bzr"}   = 1;

$opt_count_diff        = defined $opt_count_diff ? 1 : 0;

$opt_diff              = 1  if $opt_diff_alignment;

$opt_exclude_ext       = "" unless $opt_exclude_ext;

$opt_ignore_whitespace = 0  unless $opt_ignore_whitespace;

$opt_ignore_case       = 0  unless $opt_ignore_case;

$opt_lang_no_ext       = 0  unless $opt_lang_no_ext;

$opt_follow_links      = 0  unless $opt_follow_links;

$opt_diff_timeout      =10  unless $opt_diff_timeout;

$opt_csv               = 1  if $opt_csv_delimiter;

$ON_WINDOWS            = 1  if $opt_force_on_windows;

$ON_WINDOWS            = 0  if $opt_force_on_unix;

$opt_max_file_size     = 100 unless $opt_max_file_size;

my @COUNT_DIFF_ARGV        = undef;

my $COUNT_DIFF_report_file = undef;

if ($opt_count_diff) {

    die "--count-and-diff requires two arguments; got ", scalar @ARGV, "\n" 

        if scalar @ARGV != 2;

    # prefix with a dummy term so that $opt_count_diff is the

    # index into @COUNT_DIFF_ARGV to work on at each pass

    @COUNT_DIFF_ARGV = (undef, $ARGV[0],

                               $ARGV[1],

                              [$ARGV[0], $ARGV[1]]);  # 3rd pass: diff them

    $COUNT_DIFF_report_file = $opt_report_file if $opt_report_file;

}

# Options defaults:

$opt_progress_rate = 100 unless defined $opt_progress_rate;

$opt_progress_rate =   0 if     defined $opt_quiet;

if (!defined $opt_v) {

    $opt_v  = 0;

} elsif (!$opt_v) {

    $opt_v  = 1;

}

if (defined $opt_xsl) {

    $opt_xsl = $CLOC_XSL if $opt_xsl eq "1";

    $opt_xml = 1;

}

my $skip_generate_report = 0;

$opt_sql_style = 0 unless defined $opt_sql_style;

$opt_sql = 0 unless $opt_sql_style or defined $opt_sql;

if ($opt_sql eq "-" || $opt_sql eq "1") { # stream SQL output to STDOUT

    $opt_quiet            = 1;

    $skip_generate_report = 1;

    $opt_by_file          = 1;

    $opt_sum_reports      = 0;

    $opt_progress_rate    = 0;

} elsif ($opt_sql)  { # write SQL output to a file

    $opt_by_file          = 1;

    $skip_generate_report = 1;

    $opt_sum_reports      = 0;

}

if ($opt_sql_style) {

    $opt_sql_style = lc $opt_sql_style;

    if (!grep { lc $_ eq $opt_sql_style } qw ( Oracle )) {

        die "'$opt_sql_style' is not a recognized SQL style.\n";

    }

}

$opt_by_percent = '' unless defined $opt_by_percent;

if ($opt_by_percent and $opt_by_percent !~ m/^(c|cm|cb|cmb)$/i) {

    die "--by-percent must be either 'c', 'cm', 'cb', or 'cmb'\n";

}

$opt_by_percent = lc $opt_by_percent;

die $usage unless defined $opt_version         or

                  defined $opt_show_lang       or

                  defined $opt_show_ext        or

                  defined $opt_show_os         or

                  defined $opt_write_lang_def  or

                  defined $opt_list_file       or

                  defined $opt_xsl             or

                  defined $opt_explain         or

                  scalar @ARGV >= 1;

die "--diff requires two arguments; got ", scalar @ARGV, "\n" 

    if $opt_diff and scalar @ARGV != 2;

if ($opt_version) {

    printf "$VERSION\n";

    exit;

}

# 1}}}

# Step 1:  Initialize global constants.        {{{1

#

my $nFiles_Found = 0;  # updated in make_file_list

my (%Language_by_Extension, %Language_by_Script,

    %Filters_by_Language, %Not_Code_Extension, %Not_Code_Filename,

    %Language_by_File, %Scale_Factor, %Known_Binary_Archives,

    %EOL_Continuation_re,

   );

my $ALREADY_SHOWED_HEADER = 0;

my $ALREADY_SHOWED_XML_SECTION = 0;

my %Error_Codes = ( 'Unable to read'                => -1,

                    'Neither file nor directory'    => -2, 

                    'Diff error (quoted comments?)' => -3, 

                    'Diff error, exceeded timeout'  => -4, 

                  );

if ($opt_force_lang_def) {

    # replace cloc's definitions

    read_lang_def(

        $opt_force_lang_def    , #        Sample values:

        \%Language_by_Extension, # Language_by_Extension{f}    = 'Fortran 77' 

        \%Language_by_Script   , # Language_by_Script{sh}      = 'Bourne Shell'

        \%Language_by_File     , # Language_by_File{makefile}  = 'make'

        \%Filters_by_Language  , # Filters_by_Language{Bourne Shell}[0] = 

                                 #      [ 'remove_matches' , '^\s*#'  ]

        \%Not_Code_Extension   , # Not_Code_Extension{jpg}     = 1

        \%Not_Code_Filename    , # Not_Code_Filename{README}   = 1

        \%Scale_Factor         , # Scale_Factor{Perl}          = 4.0

        \%EOL_Continuation_re  , # EOL_Continuation_re{C++}    = '\\$'

        );

} else {

    set_constants(               #

        \%Language_by_Extension, # Language_by_Extension{f}    = 'Fortran 77' 

        \%Language_by_Script   , # Language_by_Script{sh}      = 'Bourne Shell'

        \%Language_by_File     , # Language_by_File{makefile}  = 'make'

        \%Filters_by_Language  , # Filters_by_Language{Bourne Shell}[0] = 

                                 #      [ 'remove_matches' , '^\s*#'  ]

        \%Not_Code_Extension   , # Not_Code_Extension{jpg}     = 1

        \%Not_Code_Filename    , # Not_Code_Filename{README}   = 1

        \%Scale_Factor         , # Scale_Factor{Perl}          = 4.0

        \%Known_Binary_Archives, # Known_Binary_Archives{.tar} = 1

        \%EOL_Continuation_re  , # EOL_Continuation_re{C++}    = '\\$'

        );

}

if ($opt_read_lang_def) {

    # augment cloc's definitions (keep cloc's where there are overlaps)

    merge_lang_def(

        $opt_read_lang_def     , #        Sample values:

        \%Language_by_Extension, # Language_by_Extension{f}    = 'Fortran 77' 

        \%Language_by_Script   , # Language_by_Script{sh}      = 'Bourne Shell'

        \%Language_by_File     , # Language_by_File{makefile}  = 'make'

        \%Filters_by_Language  , # Filters_by_Language{Bourne Shell}[0] = 

                                 #      [ 'remove_matches' , '^\s*#'  ]

        \%Not_Code_Extension   , # Not_Code_Extension{jpg}     = 1

        \%Not_Code_Filename    , # Not_Code_Filename{README}   = 1

        \%Scale_Factor         , # Scale_Factor{Perl}          = 4.0

        \%EOL_Continuation_re  , # EOL_Continuation_re{C++}    = '\\$'

        );

}

if ($opt_lang_no_ext and !defined $Filters_by_Language{$opt_lang_no_ext}) {

    die_unknown_lang($opt_lang_no_ext, "--lang-no-ext")

}

check_scale_existence(\%Filters_by_Language, \%Language_by_Extension,

                      \%Scale_Factor);

# Process command line provided extension-to-language mapping overrides.

# Make a hash of known languages in lower case for easier matching.

my %Recognized_Language_lc = (); # key = language name in lc, value = true name

foreach my $language (keys %Filters_by_Language) {

    my $lang_lc = lc $language;

    $Recognized_Language_lc{$lang_lc} = $language;

}

my %Forced_Extension = (); # file name extensions which user wants to count

my $All_One_Language = 0;  # set to !0 if --force-lang's <ext> is missing

foreach my $pair (@opt_force_lang) {

    my ($lang, $extension) = split(',', $pair);

    my $lang_lc = lc $lang;

    if (defined $extension) {

        $Forced_Extension{$extension} = $lang;

        die_unknown_lang($lang, "--force-lang")

            unless $Recognized_Language_lc{$lang_lc}; 

        $Language_by_Extension{$extension} = $Recognized_Language_lc{$lang_lc};

    } else {

        # the scary case--count everything as this language

        $All_One_Language = $Recognized_Language_lc{$lang_lc};

    }

}

foreach my $pair (@opt_script_lang) {

    my ($lang, $script_name) = split(',', $pair);

    my $lang_lc = lc $lang;

    if (!defined $script_name) {

        die "The --script-lang option requires a comma separated pair of ".

            "strings.\n";

    }

    die_unknown_lang($lang, "--script-lang")

        unless $Recognized_Language_lc{$lang_lc}; 

    $Language_by_Script{$script_name} = $Recognized_Language_lc{$lang_lc};

}

# If user provided file extensions to ignore, add these to

# the exclusion list.

foreach my $ext (map { $_ => 1 } split(/,/, $opt_exclude_ext ) ) {

    $ext = lc $ext if $ON_WINDOWS;

    $Not_Code_Extension{$ext} = 1;

}

# If SQL or --by-file output is requested, keep track of directory names

# generated by File::Temp::tempdir and used to temporarily hold the results

# of compressed archives.  Contents of the SQL table 't' will be much

# cleaner if these meaningless directory names are stripped from the front

# of files pulled from the archives.

my %TEMP_DIR = ();

my $TEMP_OFF =  0;  # Needed for --sdir; keep track of the number of

                    # scratch directories made in this run to avoid

                    # file overwrites by multiple extractions to same

                    # sdir.

# Also track locations where temporary installations, if necessary, of

# Algorithm::Diff and/or Regexp::Common are done.  Make sure these 

# directories are not counted as inputs (ref bug #80 2012-11-23).

my %TEMP_INST = ();

# invert %Language_by_Script hash to get an easy-to-look-up list of known

# scripting languages

my %Script_Language = map { $_ => 1 } values %Language_by_Script ;

# 1}}}

# Step 2:  Early exits for display, summation. {{{1

#

print_extension_info(   $opt_show_ext     ) if defined $opt_show_ext ;

print_language_info(    $opt_show_lang, '') if defined $opt_show_lang;

print_language_filters( $opt_explain      ) if defined $opt_explain  ;

exit if (defined $opt_show_ext)  or 

        (defined $opt_show_lang) or

        (defined $opt_explain);

Top_of_Processing_Loop:

# Sorry, coding purists.  Using a goto to implement --count-and-diff

# which has to do three passes over the main code, starting with

# a clean slate each time.

if ($opt_count_diff) {

    @ARGV = ( $COUNT_DIFF_ARGV[ $opt_count_diff ] );

    if ($opt_count_diff == 3) {

        $opt_diff = 1;

        @ARGV = @{$COUNT_DIFF_ARGV[ $opt_count_diff ]}; # last arg is list of list

    }

    if ($opt_report_file) { 

        # Instead of just one output file, will have three.

        # Keep their names unique otherwise results are clobbered.

        if ($opt_count_diff == 3) {

            $opt_report_file = $COUNT_DIFF_report_file . ".diff.$ARGV[0].$ARGV[1]";

        } else {

            $opt_report_file = $COUNT_DIFF_report_file . "." .  $ARGV[0];

        }

    } else {

        # STDOUT; print a header showing what it's working on

        if ($opt_count_diff == 3) {

            print "\ndiff $ARGV[0] $ARGV[1]::\n";

        } else {

            print "\n" if $opt_count_diff > 1;

            print "$ARGV[0]::\n";

        }

    }

    $ALREADY_SHOWED_HEADER      = 0;

    $ALREADY_SHOWED_XML_SECTION = 0;

}

#print "Before glob have [", join(",", @ARGV), "]\n";

@ARGV = windows_glob(@ARGV) if $ON_WINDOWS;

#print "after  glob have [", join(",", @ARGV), "]\n";

# filter out archive files if requested to do so

if (defined $opt_skip_archive) {

    my @non_archive = ();

    foreach my $candidate (@ARGV) {

        if ($candidate !~ m/${opt_skip_archive}$/) {

            push @non_archive, $candidate;

        } 

    }

    @ARGV = @non_archive;

}

if ($opt_sum_reports and $opt_diff) {

    my @results = ();

    if ($opt_list_file) { # read inputs from the list file

        my @list = read_list_file($opt_list_file);

        @results = combine_diffs(\@list);

    } else { # get inputs from the command line

        @results = combine_diffs(\@ARGV);

    }

    if ($opt_report_file) {

        write_file($opt_report_file, @results);

    } else {

        print "\n", join("\n", @results), "\n";

    }

    exit;

}

if ($opt_sum_reports) {

    my %Results = ();

    foreach my $type( "by language", "by report file" ) {

        my $found_lang = undef; 

        if ($opt_list_file) { # read inputs from the list file

            my @list = read_list_file($opt_list_file);

            $found_lang = combine_results(\@list, 

                                           $type, 

                                          \%{$Results{ $type }}, 

                                          \%Filters_by_Language );

        } else { # get inputs from the command line

            $found_lang = combine_results(\@ARGV, 

                                           $type, 

                                          \%{$Results{ $type }}, 

                                          \%Filters_by_Language );

        }

        next unless %Results;

        my $end_time = get_time();

        my @results  = generate_report($VERSION, $end_time - $start_time,

                                       $type,

                                      \%{$Results{ $type }}, \%Scale_Factor);

        if ($opt_report_file) {

            my $ext  = ".lang";

               $ext  = ".file" unless $type eq "by language";

            next if !$found_lang and  $ext  eq ".lang";

            write_file($opt_report_file . $ext, @results);

        } else {

            print "\n", join("\n", @results), "\n";

        }

    }

    exit;

}

if ($opt_write_lang_def) {

    write_lang_def($opt_write_lang_def   ,

                  \%Language_by_Extension,

                  \%Language_by_Script   ,

                  \%Language_by_File     ,

                  \%Filters_by_Language  ,

                  \%Not_Code_Extension   ,

                  \%Not_Code_Filename    ,

                  \%Scale_Factor         ,

                  \%EOL_Continuation_re  ,

                  );

    exit;

}

if ($opt_show_os) {

    if ($ON_WINDOWS) {

        print "Windows\n";

    } else {

        print "UNIX\n";

    }

    exit;

}

# 1}}}

# Step 3:  Create a list of files to consider. {{{1

#  a) If inputs are binary archives, first cd to a temp

#     directory, expand the archive with the user-given

#     extraction tool, then add the temp directory to

#     the list of dirs to process.

#  b) Create a list of every file that might contain source

#     code.  Ignore binary files, zero-sized files, and

#     any file in a directory the user says to exclude.

#  c) Determine the language for each file in the list.

#

my @binary_archive = ();

my $cwd            = cwd();

if ($opt_extract_with) {

#print "cwd main = [$cwd]\n";

    my @extract_location = ();

    foreach my $bin_file (@ARGV) {

        my $extract_dir = undef;

        if ($opt_sdir) {

            ++$TEMP_OFF;

            $extract_dir = "$opt_sdir/$TEMP_OFF";

            File::Path::rmtree($extract_dir) if     is_dir($extract_dir);

            File::Path::mkpath($extract_dir) unless is_dir($extract_dir);

        } else {

            $extract_dir = tempdir( CLEANUP => 1 );  # 1 = delete on exit

        }

        $TEMP_DIR{ $extract_dir } = 1 if $opt_sql or $opt_by_file;

        print "mkdir $extract_dir\n"  if $opt_v;

        print "cd    $extract_dir\n"  if $opt_v;

        chdir $extract_dir;

        my $bin_file_full_path = "";

        if (File::Spec->file_name_is_absolute( $bin_file )) {

            $bin_file_full_path = $bin_file;

#print "bin_file_full_path (was ful) = [$bin_file_full_path]\n";

        } else {

            $bin_file_full_path = File::Spec->catfile( $cwd, $bin_file );

#print "bin_file_full_path (was rel) = [$bin_file_full_path]\n";

        }

        my     $extract_cmd = uncompress_archive_cmd($bin_file_full_path);

        print  $extract_cmd, "\n" if $opt_v;

        system $extract_cmd;

        push @extract_location, $extract_dir;

        chdir $cwd;

    }

    # It is possible that the binary archive itself contains additional

    # files compressed the same way (true for Java .ear files).  Go

    # through all the files that were extracted, see if they are binary

    # archives and try to extract them.  Lather, rinse, repeat.

    my $binary_archives_exist = 1;

    my $count_binary_archives = 0;

    my $previous_count        = 0;

    my $n_pass                = 0;

    while ($binary_archives_exist) {

        @binary_archive = ();

        foreach my $dir (@extract_location) {

            find(\&archive_files, $dir);  # populates global @binary_archive

        }

        foreach my $archive (@binary_archive) {

            my $extract_dir = undef;

            if ($opt_sdir) {

                ++$TEMP_OFF;

                $extract_dir = "$opt_sdir/$TEMP_OFF";

                File::Path::rmtree($extract_dir) if     is_dir($extract_dir);

                File::Path::mkpath($extract_dir) unless is_dir($extract_dir);

            } else {

                $extract_dir = tempdir( CLEANUP => 1 );  # 1 = delete on exit

            }

            $TEMP_DIR{ $extract_dir } = 1 if $opt_sql or $opt_by_file;

            print "mkdir $extract_dir\n"  if $opt_v;

            print "cd    $extract_dir\n"  if $opt_v;

            chdir  $extract_dir;

            my     $extract_cmd = uncompress_archive_cmd($archive);