#!/bin/ksh93

PROG=${0##*/}

function usage {
	cat >&${1:-2} <<EOF
Usage: $PROG load [-c] [-o db] [-s repo-name-type-and-location]
	-c		Clobber DB
	-o db		DB name (default: ./db)
	-s local	Use pkg information from the running image
			(default: /var/sadm/install/contents, if non-empty,
				  else IPS pkg contents output)
	-s svr4 -    	Use SVR4 pkg information from the running image,
			with "WOS" as the repository name.
	-s ips -        Use IPS pkg information from the running image,
			with "WOS" as the repository name.

	-s ips dir      Use IPS pkg information from the given depot
			manifests dir.
	-s ips file     Use IPS pkg information from the output of "pkg
			contents -H -o action.key,pkg.name,pkg.authority
			-t file" from an installed image or repository.
	-s svr4 dir nam Use SVR4 pkg information from pkgmap files from
			the packages in the given directory.
	-s svr4 fil nam Use SVR4 pkg information from the given sadm
			contents file.

			The name argument is the name of the given
			repository.  This is used to tag the pkg
			metadata.

		Load a DB with data from SVR4 and/or IPS packaging.
		Multiple instances of the -s option, and multiple runs
		are allowed.

       $PROG analyze [-i] [-o output_dir] [db]
	-i		Interactive removal of output files
	-D		Don't delete SVR4/IPS WOS dups from the database
	-o dir		Where to put output files

		Analyze the given DB (default: ./db) for conflicts
		between repositories.
EOF
	exit 1
}

#function sql {
#	typeset line
#	if [[ $# -gt 0 ]]
#	then
#		print -rp -- $*
#	else
#		while read -r line
#		do
#			print -rp -- "$line"
#		done
#	fi
#}
#
#co_proc=false

file_pkg_list=
function cleanup {
	[[ -n "$file_pkg_list" ]] && rm -f "$file_pkg_list"
	#$co_proc || return 0
	#sql .quit
	#wait
	return 0
}

trap cleanup EXIT

# This function generates INSERT statements (wrapped in a transaction)
# from input generated by prep_src_*() (see below)
function pop_db {
	db=$1
	src=$2
	typ=$4
	(
		print "BEGIN TRANSACTION;"
		set +x
		while read a b pa t pk r
		do
			if [[ "$t" != "NULL" ]]; then
				t="'$t'"
			fi
			if [[ "$b" != "NULL" ]]; then
				b="'$b'"
			fi
			print "INSERT INTO paths VALUES ('$a', $b, '$pa', $t,
				'$pk', '$typ', '$r');"
		done < "$src"
		$debug && set -x
		print "COMMIT;"
	) | $ptime sqlite3 "$db"
}

srcs=(typeset -i num=0)

function add_src {
	nameref src="srcs.s${srcs.num}"

	set -A src $1 $2 $3
	((srcs.num += 1))

	return 0
}

## The prep_src_* functions must output lines of this form:
## <action>SP<basename>SP<path>SP<target>SP<pkgname>SP<authority>

# Parse a /var/sadm/install/contents file
function prep_src_svr4_contents {
	typeset dst

	dst=$1
	shift

	print "	processing SVR4 contents file ${1}..."
	grep ' [fev] ' "$1" | cut -d\  -f1,10 | \
		sed -e 's,^/\([^ ]*\)/\([^ ]*\) \(.*\),file \2 \1/\2 NULL \3,' \
			-e "s,\$, ${2:-WOS}," > "$dst"
	grep ' [l] ' "$1" | cut -d\  -f1,4 | \
		sed -e 's,^/\([^=]*\)=\([^ ]*\) \(.*\),link NULL \1 \2 \3,' \
			-e "s,\$, ${2:-WOS}," >> "$dst"
}

# Parse a Product directory
function prep_src_svr4_pkgs {
	typeset dst

	dst=$1
	shift

	print "	processing SVR4 Product directory ${1}..."
	find "$1" -name pkgmap | xargs grep ' [fev] ' | \
		tr : ' ' | cut -d\  -f1,5| \
		sed -e 's,^\([^ ]*\)/pkgmap \(.*\)/\(.*\)$,file \3 \2/\3 NULL \1,' \
			-e "s,\$, ${2:-WOS}," > "$dst"

	find "$1" -name pkgmap | xargs grep ' [l] ' | \
		tr : ' ' | cut -d\  -f1,5| \
		sed -e 's,^\([^ ]*\)/pkgmap \([^=].*\)/=\(.*\)$,link NULL \2 \3 \1,' \
			-e "s,\$, ${2:-WOS}," >> "$dst"
}

# Parse either a /var/sadm/install/contents file or Product dir
function prep_src_svr4 {
	if [[ "$2" = "-" ]]
	then
		prep_src_svr4_contents "$1" $v4_contents
	elif [[ -f "$2" ]]
	then
		prep_src_svr4_contents "$@"
	elif [[ -d "$2" ]]
	then
		prep_src_svr4_pkgs "$@"
	else
		usage
	fi
}

# Parse IPS pkg meta-data
function prep_src_ips {
	typeset dst p n a b

	dst=$1
	shift

	if [[ "$1" = "-" ]]
	then
		print "	processing IPS metadata for current image..."
		pkg contents -H -o action.key,pkg.name,pkg.authority -t file | 
			(
			set +x
			while read p n a
			do
				print -r "file ${p##*/} $p NULL $n $a"
			done ) > "$dst"
			#sed -e 's,^\([^ ]*\)/\([^/]*\) \([^ ]*\) \(.*\)$,\2 \1/\2 \3 \4,'
		pkg contents -H -o path,target,pkg.name,pkg.authority -t link |
			(
			set +x
			while read p t n a
			do
				print -r "link NULL $p $t $n $a"
			done ) >> "$dst"
	elif [[ -f "$1" ]]
	then
		print "	processing IPS metadata in ${1}..."
		set +x
		while read p n a
		do
			print -r "file ${p##*/} $p NULL $n $a"
		done < "$1" > "$dst"
		$debug && set -x
	else
		# This looks through a dir that contains dirs named
		# after pkgs and which contain manifests.  I.e., like
		# /var/pkg/repo/pkg.
		#
		# The find ...| xargs grep ... pipe produces lines like:
		# <pkgname>/<manifest-filename>:file...path=<path>...
		print "	processing IPS metadata in depot directory ${1}..."
		(
		cd "$1" > /dev/null
		find . -type f | xargs grep ^file /dev/null | \
			sed -e 's,^\./\([^/]*\)/[^:]*:\(.*\)$,\1:\2,' \
			-e 's,^\([^:]*\):file.*path=\([^ ]*\) .*$,\2 \1,' \
			-e 's,^\([^ ]*\)/\([^ ]*\) \([^/]*\)$,file \2 \1/\2 NULL \3,' \
			-e "s,\$, ${2:-opensolaris.org}," > "$dst"
		find . -type f | xargs grep ^link /dev/null | \
			sed -e 's,^\./\([^/]*\)/[^:]*:\(.*\)$,\1:\2,' \
			-e 's,^\([^:]*\):link.*path=\([^ ]*\) target=\([^ ]*\),link NULL \2 \3 \1,' \
			-e "s,\$, ${2:-opensolaris.org}," >> "$dst"
		)
	fi
}

function create_views {
	db=$1
	$ptime sqlite3 "$db" <<EOF
		DROP VIEW IF EXISTS conf_paths;
		DROP VIEW IF EXISTS conf_basenames;
		DROP VIEW IF EXISTS conf_sonames;
		CREATE VIEW conf_paths AS SELECT c, path, pkg FROM
			(SELECT count(path) AS c, path,
				group_concat(pkg) AS pkg
				FROM paths
				WHERE action = 'file' OR action = 'link'
				GROUP BY path)
			WHERE c > 1
			ORDER BY c DESC;
		CREATE VIEW conf_basenames AS SELECT c, basename, pkg FROM
			(SELECT count(DISTINCT pkg) AS c, basename,
				group_concat(DISTINCT pkg) AS pkg
				FROM paths
				WHERE action = 'file'
				GROUP BY basename)
			WHERE c > 1
			ORDER BY c DESC;
		CREATE VIEW conf_sonames AS SELECT c, soname, pkg FROM
			(SELECT count(DISTINCT pkg) AS c,
				rtrim(basename, '.0123456789') AS soname,
				group_concat(DISTINCT pkg) AS pkg
				FROM paths
				WHERE basename like 'lib%.so.%'
					AND action = 'file'
				GROUP BY rtrim(basename, '.0123456789'))
			WHERE c > 1
			ORDER BY c DESC;
EOF
}

typeset -l typ
v4_contents=/var/sadm/install/contents
def_ips_data="$(/bin/pwd)/manifests"
[[ -d "$def_ips_data" ]] || def_ips_data=/var/pkg/repo/pkg
def_wos_data="/var/sadm/install/contents"
interact=false
output_dir=$(/bin/pwd)
db=$(/bin/pwd)/db
debug=false
load=false
analyze=false
clobber_db=false
delete_dups=:
conf_basename_cutoff=100

# Yes, this should be made to use getopts.
while [[ $# -gt 0 ]]
do
	case "$1" in
	load)
		load=:
		shift
		while [[ $# -gt 0 ]]
		do
			case "$1" in
			-x)	debug=:
				typeset -ft $(typeset +f|awk '{print $1}')
				set -x;;
			-c)	clobber_db=:;;
			-o)	db=$2; shift;;
			-s)
				if [[ "$2" = local ]]
				then
					if [[ -s $v4_contents ]]
					then
						add_src svr4 $v4_contents WOS
					else
						add_src ips - -
					fi
					shift
				elif [[ "$2" = svr4 && "$3" = "-" ]]
				then
					add_src $2 - WOS
					shift 2
				elif [[ "$2" = ips ]]
				then
					add_src $2 "$3" -
					shift 2
				else
					[[ $# -ge 4 || "$2" != svr4 ]] || usage
					add_src "$2" "$3" "$4"
					shift 3
				fi ;;
			*) usage 1;;
			esac
			shift
		done
		;;
	analyze)
		analyze=:
		shift
		while [[ $# -gt 0 ]]
		do
			case "$1" in
			-x)	debug=:
				typeset -ft $(typeset +f|awk '{print $1}')
				set -x;;
			-D) delete_dups=false;;
			-i) interact=:;;
			-o) output_dir=$2; db=$output_dir/db; shift;;
			-*) usage 1;;
			*)	db=$1; shift; break;;
			esac
			shift
		done
		[[ $# -gt 0 ]] && usage 1
		if [[ ! -f $db ]]
		then
			print -u2 "Error: must load the data first"
			usage
		fi
		;;
	*) usage 1;;
	esac
done
[[ $# -gt 0 ]] && usage
$load || $analyze || usage

if $clobber_db && [[ -f "$db" ]]
then
	rm -f "$db"
fi

if $load
then
	if ((${srcs.num} == 0))
	then
		usage
	fi

	print "Loading pkg data into $db..."

	file_pkg_list=$(mktemp /tmp/file_pkg_list_XXXXX)
	if [[ -z "$file_pkg_list" ]]
	then
		print -u2 "Error: Could not create temporary files"
		exit 1
	fi

	sqlite3 "$db" "CREATE TABLE IF NOT EXISTS paths(
		action TEXT NOT NULL,
		basename TEXT,
		path TEXT NOT NULL,
		target TEXT,
		pkg TEXT NOT NULL,
		pkg_typ TEXT NOT NULL,
		repo TEXT NOT NULL
		);"

	sqlite3 "$db" "DROP INDEX IF EXISTS basename;"
	sqlite3 "$db" "DROP INDEX IF EXISTS path;"

	integer i=0
	while [[ $i < ${srcs.num} ]]
	do
		nameref src="srcs.s${i}"
		((i+=1))

		typ=${src[0]}
		loc=${src[1]}
		repo=${src[2]}

		if [[ -z "$typ" ]]
		then
			print -u2 "Error: internal error"
			exit 2
		fi

		prep_src_$typ "$file_pkg_list" "$loc" "$repo"
		pop_db "$db" "$file_pkg_list" "$typ"
		cat /dev/null > $file_pkg_list
	done

	print "	creating various views ..."
	create_views "$db"
fi

if ! $analyze
then
	exit 0
fi

set -A ofiles -- conf_pkgs_by_path conf_pkgs_by_basename conf_pkgs_by_soname \
	conf_paths conf_pkg_pairs conf_basenames conf_sonames \
	conf_pkg_pairs_similar
for i in "${ofiles[@]}"
do
	[[ -f "$output_dir/$i" ]] || continue
	if $interact
	then
		typeset -l ans
		print -n "Clean up existing analysis? (y/n) "
		read ans
		case "$ans" in
			y|yes)
				for i in "${files[@]}"
				do
					rm -f "$output_dir/$i"
				done
				;;
			*) exit 1;;
		esac
	else
		for i in "${files[@]}"
		do
			rm -f "$output_dir/$i"
		done
	fi
	break
done

print "Analyzing data..."

print "	creating indexes..."
$ptime sqlite3 "$db" <<EOF
	CREATE INDEX IF NOT EXISTS basename on paths (basename);
	CREATE INDEX IF NOT EXISTS path on paths (path);
EOF

if $delete_dups
then
print "	removing dups between WOS and OpenSolaris..."
$ptime sqlite3 "$db" <<EOF
	-- Delete dups from OpenSolaris where the path is dup and the other pkg
	-- is an SVR4 pkg from the WOS
	DELETE FROM paths WHERE repo = 'opensolaris.org' AND path in
		(SELECT p1.path FROM paths p1 JOIN paths p2 USING(path, action)
		WHERE
			p1.repo = 'opensolaris.org' AND p2.repo = 'WOS' AND
			p1.pkg_typ = 'ips' AND (p1.pkg = rtrim(p2.pkg, 'ru'))
			);
		--(SELECT p1.path FROM paths p1 JOIN paths p2 USING(path)
		--WHERE
		--	((p1.repo = 'WOS' AND p2.repo = 'opensolaris.org') OR
		--	(p2.repo = 'WOS' AND p1.repo = 'opensolaris.org')) AND
		--	(p1.pkg = p2.pkg OR
		--	(p1.pkg_typ = 'ips' AND (p1.pkg = rtrim(p2.pkg, 'ru')))
		--	OR
		--	(p2.pkg_typ = 'ips' AND (p2.pkg = rtrim(p1.pkg, 'ru'))))
		--	);
EOF
fi

print "	creating various tables ..."
$ptime sqlite3 "$db" <<EOF
	DROP TABLE IF EXISTS conf_pkgs_by_path;
	DROP TABLE IF EXISTS conf_pkgs_by_basename;
	DROP TABLE IF EXISTS conf_pkgs_by_soname;
	DROP TABLE IF EXISTS conf_pkg_pairs;
	DROP TABLE IF EXISTS basenames2pkg;
	DROP TABLE IF EXISTS sonames;
	DROP TABLE IF EXISTS sonames2pkg;
	CREATE TABLE conf_pkgs_by_path(pkg_list, UNIQUE (pkg_list));
	CREATE TABLE conf_pkgs_by_basename(pkg_list, UNIQUE (pkg_list));
	CREATE TABLE conf_pkgs_by_soname(pkg_list, UNIQUE (pkg_list));
	CREATE TABLE conf_pkg_pairs(pkg1 TEXT NOT NULL,
		pkg2 TEXT NOT NULL, c INTEGER, UNIQUE (pkg1, pkg2));
	CREATE TABLE basenames2pkg(basename TEXT NOT NULL, pkg TEXT NOT NULL);
	CREATE TABLE sonames(soname TEXT NOT NULL);
	CREATE TABLE sonames2pkg(soname TEXT NOT NULL,
		pkg TEXT NOT NULL,
		pkg_typ TEXT NOT NULL,
		repo TEXT NOT NULL
	);
EOF

print "	analyzing data, populating tables..."
$ptime sqlite3 "$db" <<EOF
	BEGIN TRANSACTION;
	INSERT INTO basenames2pkg
		SELECT DISTINCT basename, pkg FROM paths WHERE action = 'file';
	INSERT INTO sonames
		SELECT DISTINCT rtrim(basename, '.0123456789') as soname
		FROM paths
		WHERE basename like 'lib%.so.%' AND action = 'file';
	INSERT INTO sonames2pkg SELECT DISTINCT
		rtrim(basename, '.0123456789') AS soname, pkg, pkg_typ, repo
		FROM paths
		WHERE basename like 'lib%.so.%' AND action = 'file';
	INSERT OR IGNORE INTO conf_pkgs_by_path
		SELECT pkg FROM conf_paths;
	INSERT OR IGNORE INTO conf_pkgs_by_basename
		SELECT pkg FROM conf_basenames
		WHERE c < $conf_basename_cutoff;
	INSERT OR IGNORE INTO conf_pkgs_by_soname
		SELECT pkg FROM conf_sonames;
	INSERT INTO conf_pkg_pairs SELECT pkg1, pkg2, count(*) as c FROM
		(SELECT p1.pkg AS pkg1, p2.pkg AS pkg2
			FROM paths AS p1 JOIN paths AS p2 USING (path, action)
			WHERE pkg1 != pkg2
			GROUP BY p1.path)
		GROUP BY pkg1, pkg2 ORDER BY c DESC;
	COMMIT;
EOF

print "	producing analysis..."
$ptime sqlite3 "$db" <<EOF
.headers on
	SELECT count(*) AS "Total file paths"
		FROM paths WHERE action = 'file';
	SELECT count(*) AS "Total distinct file paths"
		FROM (SELECT DISTINCT path FROM paths WHERE action = 'file');
	SELECT count(*) AS "Total distinct file basenames"
		FROM (SELECT DISTINCT basename FROM paths WHERE action = 'file');
	SELECT count(*) AS "Total distinct library sonames"
		FROM sonames;
	SELECT count(*) AS "Conflicting paths" FROM conf_paths;
.output $output_dir/conf_paths
	SELECT c, path, pkg FROM conf_paths;
.output stdout
	SELECT count(*) AS "Conflicting basenames" FROM conf_basenames
		WHERE c < $conf_basename_cutoff;
.output $output_dir/conf_basenames
	SELECT c, basename, pkg FROM conf_basenames
		WHERE c < $conf_basename_cutoff;
.output stdout
	SELECT count(*) AS "Conflicting sonames" FROM conf_sonames;
.output $output_dir/sonames2pkg
	SELECT soname, pkg FROM sonames2pkg;
.output $output_dir/conf_pkgs_by_path
	SELECT * FROM conf_pkgs_by_path;
.output $output_dir/conf_pkgs_by_basename
	SELECT * FROM conf_pkgs_by_basename;
.output $output_dir/conf_pkgs_by_soname
	SELECT * FROM conf_pkgs_by_soname;
.output $output_dir/conf_pkg_pairs
	SELECT * FROM conf_pkg_pairs;
.output $output_dir/conf_pkg_pairs_similar
	SELECT * FROM conf_pkg_pairs
		WHERE lower(pkg1) = lower(pkg2);
.headers off
	SELECT '	PKG names differing in case and prefix';
.headers on
	SELECT * FROM conf_pkg_pairs WHERE
		(length(pkg2) > length(pkg1) AND
		lower(pkg1) = substr(lower(pkg2), 0, length(pkg1))) OR
		(length(pkg1) > length(pkg2) AND
		lower(pkg2) = substr(lower(pkg1), 0, length(pkg2)));
.output stdout
	SELECT count(*) AS "Conflicting PKG lists" FROM conf_pkgs_by_path;
	SELECT count(*) AS "All PKG pairs with one or more conflicting paths"
		FROM conf_pkg_pairs;
	SELECT count(*) AS "Conflicting PKG pairs differing only in case"
		FROM conf_pkg_pairs WHERE lower(pkg1) = lower(pkg2);
	SELECT count(*) AS
		"Conflicting PKG pairs differing in case and prefix or suffix"
		FROM conf_pkg_pairs
		WHERE	(length(pkg2) > length(pkg1) AND
			lower(pkg1) = substr(lower(pkg2), 0, length(pkg1))) OR
			(length(pkg1) > length(pkg2) AND
			lower(pkg2) = substr(lower(pkg1), 0, length(pkg2)));
EOF

cat <<EOF
	SQLite3 DB created at $db; you can use this for
	further manual analysis.
EOF

exit 0